Add option to preserve data in test clusters (#65400)

Author: mark-vieira

TESTING.asciidoc

@@ -82,12 +82,13 @@ password: `elastic-password`.
 
 ==== Other useful arguments
 
-In order to start a node with a different max heap space add: `-Dtests.heap.size=4G`
-In order to disable assertions add: `-Dtests.asserts=false`
-In order to use a custom data directory: `--data-dir=/tmp/foo`
-In order to remotely attach a debugger to the process: `--debug-jvm`
-In order to set a different keystore password: `--keystore-password`
-In order to set an Elasticsearch setting, provide a setting with the following prefix: `-Dtests.es.`
+- In order to start a node with a different max heap space add: `-Dtests.heap.size=4G`
+- In order to disable assertions add: `-Dtests.asserts=false`
+- In order to use a custom data directory: `--data-dir=/tmp/foo`
+- In order to preserve data in between executions: `--preserve-data`
+- In order to remotely attach a debugger to the process: `--debug-jvm`
+- In order to set a different keystore password: `--keystore-password`
+- In order to set an Elasticsearch setting, provide a setting with the following prefix: `-Dtests.es.`
 
 === Test case filtering.
 
@@ -308,15 +309,15 @@ The REST tests are run automatically when executing the "./gradlew check" comman
 YAML REST tests use the following command (modules and plugins may also include YAML REST tests):
 
 ---------------------------------------------------------------------------
-./gradlew :rest-api-spec:yamlRestTest 
+./gradlew :rest-api-spec:yamlRestTest
 ---------------------------------------------------------------------------
 
 A specific test case can be run with the following command:
 
 ---------------------------------------------------------------------------
 ./gradlew ':rest-api-spec:yamlRestTest' \
   --tests "org.elasticsearch.test.rest.ClientYamlTestSuiteIT" \
-  -Dtests.method="test {p0=cat.segments/10_basic/Help}" 
+  -Dtests.method="test {p0=cat.segments/10_basic/Help}"
 ---------------------------------------------------------------------------
 
 The YAML REST tests support all the options provided by the randomized runner, plus the following:
@@ -334,14 +335,14 @@ Java REST tests can be run with the "javaRestTest" task.
 
 For example :
 ---------------------------------------------------------------------------
-./gradlew :modules:mapper-extras:javaRestTest 
+./gradlew :modules:mapper-extras:javaRestTest
 ---------------------------------------------------------------------------
 
 A specific test case can be run with the following syntax (fqn.test {params}):
 
 ---------------------------------------------------------------------------
 ./gradlew ':modules:mapper-extras:javaRestTest' \
-  --tests "org.elasticsearch.index.mapper.TokenCountFieldMapperIntegrationIT.testSearchByTokenCount {storeCountedFields=true loadCountedFields=false}" 
+  --tests "org.elasticsearch.index.mapper.TokenCountFieldMapperIntegrationIT.testSearchByTokenCount {storeCountedFields=true loadCountedFields=false}"
 ---------------------------------------------------------------------------
 
 yamlRestTest's and javaRestTest's are easy to identify, since they are found in a

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchCluster.java

@@ -282,6 +282,16 @@ public void jvmArgs(String... values) {
         nodes.all(each -> each.jvmArgs(values));
     }
 
+    @Internal
+    public boolean isPreserveDataDir() {
+        return nodes.stream().anyMatch(node -> node.isPreserveDataDir());
+    }
+
+    @Override
+    public void setPreserveDataDir(boolean preserveDataDir) {
+        nodes.all(each -> each.setPreserveDataDir(preserveDataDir));
+    }
+
     @Override
     public void freeze() {
         nodes.forEach(ElasticsearchNode::freeze);

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java

@@ -127,7 +127,6 @@ public class ElasticsearchNode implements TestClusterConfiguration {
     private final FileSystemOperations fileSystemOperations;
     private final ArchiveOperations archiveOperations;
     private final ExecOperations execOperations;
-
     private final AtomicBoolean configurationFrozen = new AtomicBoolean(false);
     private final Path workingDir;
 
@@ -167,6 +166,7 @@ public class ElasticsearchNode implements TestClusterConfiguration {
     private String transportPort = "0";
     private Path confPathData;
     private String keystorePassword = "";
+    private boolean preserveDataDir = false;
 
     ElasticsearchNode(
         String path,
@@ -421,6 +421,17 @@ public Path getConfigDir() {
         return configFile.getParent();
     }
 
+    @Override
+    @Input
+    public boolean isPreserveDataDir() {
+        return preserveDataDir;
+    }
+
+    @Override
+    public void setPreserveDataDir(boolean preserveDataDir) {
+        this.preserveDataDir = preserveDataDir;
+    }
+
     @Override
     public void freeze() {
         requireNonNull(testDistribution, "null testDistribution passed when configuring test cluster `" + this + "`");
@@ -453,7 +464,13 @@ public synchronized void start() {
                 logToProcessStdout("Configuring working directory: " + workingDir);
                 // make sure we always start fresh
                 if (Files.exists(workingDir)) {
-                    fileSystemOperations.delete(d -> d.delete(workingDir));
+                    if (preserveDataDir) {
+                        Files.list(workingDir)
+                            .filter(path -> path.equals(confPathData) == false)
+                            .forEach(path -> fileSystemOperations.delete(d -> d.delete(path)));
+                    } else {
+                        fileSystemOperations.delete(d -> d.delete(workingDir));
+                    }
                 }
                 isWorkingDirConfigured = true;
             }

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/RunTask.java

@@ -46,6 +46,8 @@ public class RunTask extends DefaultTestClustersTask {
 
     private Boolean debug = false;
 
+    private Boolean preserveData = false;
+
     private Path dataDir = null;
 
     private String keystorePassword = "";
@@ -65,6 +67,16 @@ public void setDataDir(String dataDirStr) {
         dataDir = Paths.get(dataDirStr).toAbsolutePath();
     }
 
+    @Input
+    public Boolean getPreserveData() {
+        return preserveData;
+    }
+
+    @Option(option = "preserve-data", description = "Preserves data directory contents (path provided to --data-dir is always preserved)")
+    public void setPreserveData(Boolean preserveData) {
+        this.preserveData = preserveData;
+    }
+
     @Option(option = "keystore-password", description = "Set the elasticsearch keystore password")
     public void setKeystorePassword(String password) {
         keystorePassword = password;
@@ -113,6 +125,7 @@ public void beforeStart() {
             httpPort++;
             cluster.getFirstNode().setTransportPort(String.valueOf(transportPort));
             transportPort++;
+            cluster.setPreserveDataDir(preserveData);
             for (ElasticsearchNode node : cluster.getNodes()) {
                 additionalSettings.forEach(node::setting);
                 if (dataDir != null) {

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/StandaloneRestIntegTestTask.java

@@ -66,6 +66,13 @@ public StandaloneRestIntegTestTask() {
                     .filter(task -> task != this)
                     .anyMatch(task -> Collections.disjoint(task.getClusters(), getClusters()) == false)
             );
+
+        this.getOutputs()
+            .doNotCacheIf(
+                "Caching disabled for this task since it is configured to preserve data directory",
+                // Don't cache the output of this task if it's not running from a clean data directory.
+                t -> getClusters().stream().anyMatch(cluster -> cluster.isPreserveDataDir())
+            );
     }
 
     @Override

buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterConfiguration.java

@@ -87,6 +87,10 @@ public interface TestClusterConfiguration {
 
     void jvmArgs(String... values);
 
+    boolean isPreserveDataDir();
+
+    void setPreserveDataDir(boolean preserveDataDir);
+
     void freeze();
 
     void start();

x-pack/plugin/eql/qa/correctness/README.md

@@ -11,11 +11,11 @@ the queries and asserts the results that are provided along with the query state
 ### Running the tests
 
 To be able to run the tests locally, one should set the environmental variable `eql_test_credentials_file` pointing to
-a local file holding the service account credentials which allow access to the gcs bucket where the dataset resides. 
+a local file holding the service account credentials which allow access to the gcs bucket where the dataset resides.
 E.g.:
 ```shell script
 export eql_test_credentials_file=/Users/username/credentials.gcs.json
-``` 
+```
 
 To run the tests you can issue:
 ```shell script
@@ -46,7 +46,7 @@ queries executed, e.g.:*
 
 #### Run a specific query
 
-If one wants to run just one query from the set, needs to do it with following command by replacing `<queryNo>` (which 
+If one wants to run just one query from the set, needs to do it with following command by replacing `<queryNo>` (which
 can be found in queries.toml file) with the desired number of the query:
 
 ```shell script
@@ -66,7 +66,7 @@ or
 ./gradlew ':x-pack:plugin:eql:qa:correctness:javaRestTest' --tests "org.elasticsearch.xpack.eql.EsEQLCorrectnessIT.test {<queryNo>}" -Dtests.eql_correctness_debug=true
 ```
 
-### Run an ES node manually and run the tests against it 
+### Run an ES node manually and run the tests against it
 
 If one wants to run an ES node manually (most probably to be able to debug the server), needs to run the following:
 
@@ -76,7 +76,7 @@ If one wants to run an ES node manually (most probably to be able to debug the s
 
 **Set the `eql_test_credentials_file` environmental variable correctly in the shell before running the command above,**
 
-Once the ES node is up and running, the data can be restored from the snapshot by running the `main` of the 
+Once the ES node is up and running, the data can be restored from the snapshot by running the `main` of the
 `EqlDataLoader` class.
 
 Once the data is loaded, a specific query can be run against the running ES node with:
@@ -85,3 +85,10 @@ Once the data is loaded, a specific query can be run against the running ES node
 ```
 
 **Set the `eql_test_credentials_file` environmental variable correctly in the shell before running the command above,**
+
+#### Preserve data across node restarts
+If you'd like to preserve the restored index and avoid the network download and delay of restoring them on every run of the node,
+you can set the `eql.test.preserve.data` system property, e.g.:
+```shell script
+./gradlew :x-pack:plugin:eql:qa:correctness:javaRestTest -Deql.test.preserve.data=true
+```

x-pack/plugin/eql/qa/correctness/build.gradle

@@ -20,14 +20,26 @@ dependencies {
   javaRestTestImplementation 'io.ous:jtoml:2.0.0'
 }
 
-File serviceAccountFile = (System.getenv("eql_test_credentials_file") ?: System.getProperty("eql.test.credentials.file")) as File
+File serviceAccountFile = providers.environmentVariable('eql_test_credentials_file')
+  .forUseAtConfigurationTime()
+  .orElse(providers.systemProperty('eql.test.credentials.file').forUseAtConfigurationTime())
+  .map { s -> new File(s)}
+  .getOrNull()
+
+Boolean preserveData = providers.systemProperty('eql.test.preserve.data')
+  .forUseAtConfigurationTime()
+  .map { s -> Boolean.parseBoolean(s) }
+  .getOrElse(false)
 
 testClusters {
   all {
     plugin ':plugins:repository-gcs'
     if (serviceAccountFile) {
       keystore 'gcs.client.eql_test.credentials_file', serviceAccountFile
     }
+    if (preserveData) {
+      preserveDataDir = true
+    }
     testDistribution = 'DEFAULT'
     setting 'xpack.license.self_generated.type', 'basic'
     jvmArgs '-Xms4g', '-Xmx4g'
@@ -46,6 +58,6 @@ tasks.named('javaRestTest').configure {
 }
 
 tasks.register("runEqlCorrectnessNode", RunTask) {
-  useCluster testClusters.runTask;
+  useCluster testClusters.runTask
   description = 'Runs elasticsearch in the foreground with gcs plugin and keystore credentials'
 }