From 1e1bc67c26eeeaa218be4a82928c0ecda4817ac6 Mon Sep 17 00:00:00 2001 From: Andy Bristol Date: Tue, 5 Jun 2018 16:11:33 -0700 Subject: [PATCH 1/5] [test] port archive distribution packaging tests Recreates the rest of the bats packaging tests for the tar distribution in the java packaging test project, with support for both tar and zip packaging, both oss and default flavors, and on Linux and Windows. Most tests are followed fairly closely, some have either been dropped if unnecessary or folded into others if convenient. --- .../gradle/vagrant/VagrantTestPlugin.groovy | 10 +- qa/vagrant/build.gradle | 20 ++ .../packaging/test/ArchiveTestCase.java | 245 ++++++++++++++++++ .../packaging/util/Archives.java | 171 +++++++++--- .../packaging/util/FileUtils.java | 31 ++- .../packaging/util/Installation.java | 13 + .../packaging/util/ServerUtils.java | 125 +++++++++ .../elasticsearch/packaging/util/Shell.java | 4 + .../packaging/tests/20_tar_package.bats | 178 ------------- 9 files changed, 576 insertions(+), 221 deletions(-) create mode 100644 qa/vagrant/src/main/java/org/elasticsearch/packaging/util/ServerUtils.java delete mode 100644 qa/vagrant/src/test/resources/packaging/tests/20_tar_package.bats diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy index 455d30f95db32..d4d1d857e90d4 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy @@ -546,9 +546,15 @@ class VagrantTestPlugin implements Plugin { javaPackagingTest.command = 'ssh' javaPackagingTest.args = ['--command', 'sudo bash "$PACKAGING_TESTS/run-tests.sh"'] } else { + // powershell sessions run over winrm always run as administrator, whether --elevated is passed or not. however + // remote sessions have some restrictions on what they can do, such as impersonating another user (or the same user + // without administrator elevation), which we need to do for these tests. passing --elevated runs the session + // as a scheduled job locally on the vm as a true administrator to get around this limitation + // + // https://github.com/hashicorp/vagrant/blob/9c299a2a357fcf87f356bb9d56e18a037a53d138/plugins/communicators/winrm/communicator.rb#L195-L225 + // https://devops-collective-inc.gitbooks.io/secrets-of-powershell-remoting/content/manuscript/accessing-remote-computers.html javaPackagingTest.command = 'winrm' - // winrm commands run as administrator - javaPackagingTest.args = ['--command', 'powershell -File "$Env:PACKAGING_TESTS/run-tests.ps1"'] + javaPackagingTest.args = ['--elevated', '--command', 'powershell -File "$Env:PACKAGING_TESTS/run-tests.ps1"'] } TaskExecutionAdapter javaPackagingReproListener = createReproListener(project, javaPackagingTest.path) diff --git a/qa/vagrant/build.gradle b/qa/vagrant/build.gradle index 23d171f312512..704136eb4cf27 100644 --- a/qa/vagrant/build.gradle +++ b/qa/vagrant/build.gradle @@ -31,6 +31,12 @@ dependencies { compile "org.hamcrest:hamcrest-core:${versions.hamcrest}" compile "org.hamcrest:hamcrest-library:${versions.hamcrest}" + compile "org.apache.httpcomponents:httpcore:${versions.httpcore}" + compile "org.apache.httpcomponents:httpclient:${versions.httpclient}" + compile "org.apache.httpcomponents:fluent-hc:${versions.httpclient}" + compile "commons-codec:commons-codec:${versions.commonscodec}" + compile "commons-logging:commons-logging:${versions.commonslogging}" + compile project(':libs:core') // pulls in the jar built by this project and its dependencies @@ -73,3 +79,17 @@ tasks.test.enabled = false // this project doesn't get published tasks.dependencyLicenses.enabled = false tasks.dependenciesInfo.enabled = false + +tasks.thirdPartyAudit.excludes = [ + //commons-logging optional dependencies + 'org.apache.avalon.framework.logger.Logger', + 'org.apache.log.Hierarchy', + 'org.apache.log.Logger', + 'org.apache.log4j.Category', + 'org.apache.log4j.Level', + 'org.apache.log4j.Logger', + 'org.apache.log4j.Priority', + //commons-logging provided dependencies + 'javax.servlet.ServletContextEvent', + 'javax.servlet.ServletContextListener' +] diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java index ab4a11922cc21..bff5b3f9aece7 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java @@ -19,6 +19,12 @@ package org.elasticsearch.packaging.test; +import org.apache.http.client.fluent.Request; +import org.elasticsearch.packaging.util.Archives; +import org.elasticsearch.packaging.util.Platforms; +import org.elasticsearch.packaging.util.ServerUtils; +import org.elasticsearch.packaging.util.Shell; +import org.elasticsearch.packaging.util.Shell.Result; import org.junit.Before; import org.junit.BeforeClass; import org.junit.FixMethodOrder; @@ -28,9 +34,32 @@ import org.elasticsearch.packaging.util.Distribution; import org.elasticsearch.packaging.util.Installation; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.stream.Stream; + +import static java.util.stream.Collectors.joining; +import static org.elasticsearch.packaging.util.Archives.ARCHIVE_OWNER; import static org.elasticsearch.packaging.util.Cleanup.cleanEverything; import static org.elasticsearch.packaging.util.Archives.installArchive; import static org.elasticsearch.packaging.util.Archives.verifyArchiveInstallation; +import static org.elasticsearch.packaging.util.FileMatcher.Fileness.File; +import static org.elasticsearch.packaging.util.FileMatcher.file; +import static org.elasticsearch.packaging.util.FileMatcher.p660; +import static org.elasticsearch.packaging.util.FileUtils.append; +import static org.elasticsearch.packaging.util.FileUtils.cp; +import static org.elasticsearch.packaging.util.FileUtils.getTempDir; +import static org.elasticsearch.packaging.util.FileUtils.mkdir; +import static org.elasticsearch.packaging.util.FileUtils.rm; +import static org.elasticsearch.packaging.util.ServerUtils.makeRequest; +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.CoreMatchers.notNullValue; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.isEmptyString; +import static org.junit.Assert.assertTrue; +import static org.junit.Assume.assumeThat; import static org.junit.Assume.assumeTrue; /** @@ -61,4 +90,220 @@ public void test10Install() { installation = installArchive(distribution()); verifyArchiveInstallation(installation, distribution()); } + + @Test + public void test20PluginsListWithNoPlugins() { + assumeThat(installation, is(notNullValue())); + + final Shell sh = new Shell(); + final Result r = Platforms.WINDOWS + ? sh.powershell(installation.bin("elasticsearch-plugin.bat") + " list") + : sh.bash(installation.bin("elasticsearch-plugin") + " list"); + + assertThat(r.stdout, isEmptyString()); + } + + @Test + public void test30AbortWhenJavaMissing() { + assumeThat(installation, is(notNullValue())); + + final Shell sh = new Shell(); + if (Platforms.WINDOWS) { + // on windows, removing java from PATH and removing JAVA_HOME is less involved than changing the permissions of the java + // executable. we also don't check permissions in the windows scripts anyway + final String originalPath = sh.powershell("$Env:PATH").stdout.trim(); + final String newPath = Arrays.stream(originalPath.split(";")) + .filter(path -> path.contains("Java") == false) + .collect(joining(";")); + + final String javaHome = sh.powershell("$Env:JAVA_HOME").stdout.trim(); + + // note the lack of a $ when clearing the JAVA_HOME env variable - with a $ it deletes the java home directory + // https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/providers/environment-provider?view=powershell-6 + // + // this won't persist to another session so we don't have to reset anything + final Result runResult = sh.powershellIgnoreExitCode( + "$Env:PATH = '" + newPath + "'; " + + "Remove-Item Env:JAVA_HOME; " + + installation.bin("elasticsearch.bat") + ); + + assertThat(runResult.exitCode, is(1)); + assertThat(runResult.stderr, containsString("could not find java; set JAVA_HOME or ensure java is in PATH")); + + } else { + final String javaPath = sh.bash("which java").stdout.trim(); + sh.bash("chmod -x '" + javaPath + "'"); + final Result runResult = sh.bashIgnoreExitCode(installation.bin("elasticsearch").toString()); + sh.bash("chmod +x '" + javaPath + "'"); + + assertThat(runResult.exitCode, is(1)); + assertThat(runResult.stdout, containsString("could not find java; set JAVA_HOME or ensure java is in PATH")); + } + } + + @Test + public void test40CreateKeystoreManually() { + assumeThat(installation, is(notNullValue())); + + final Shell sh = new Shell(); + if (Platforms.WINDOWS) { + // this is a hack around the fact that we can't run a command in the same session as the same user but not as administrator. + // the keystore ends up being owned by the Administrators group, so we manually set it to be owned by the vagrant user here. + // from the server's perspective the permissions aren't really different, this is just to reflect what we'd expect in the tests. + // when we run these commands as a role user we won't have to do this + sh.powershell( + installation.bin("elasticsearch-keystore.bat") + " create; " + + "$account = New-Object System.Security.Principal.NTAccount 'vagrant'; " + + "$acl = Get-Acl '" + installation.config("elasticsearch.keystore") + "'; " + + "$acl.SetOwner($account); " + + "Set-Acl '" + installation.config("elasticsearch.keystore") + "' $acl" + ); + } else { + sh.bash("sudo -u " + ARCHIVE_OWNER + " " + installation.bin("elasticsearch-keystore") + " create"); + } + + assertThat(installation.config("elasticsearch.keystore"), file(File, ARCHIVE_OWNER, ARCHIVE_OWNER, p660)); + + final Result r = Platforms.WINDOWS + ? sh.powershell(installation.bin("elasticsearch-keystore.bat") + " list") + : sh.bash("sudo -u " + ARCHIVE_OWNER + " " + installation.bin("elasticsearch-keystore") + " list"); + assertThat(r.stdout, containsString("keystore.seed")); + + // cleanup for next test + rm(installation.config("elasticsearch.keystore")); + } + + @Test + public void test50StartAndStop() { + assumeThat(installation, is(notNullValue())); + + Archives.runElasticsearch(installation); + + final String gcLogName = Platforms.LINUX + ? "gc.log.0.current" + : "gc.log"; + assertTrue("gc logs exist", Files.exists(installation.logs.resolve(gcLogName))); + ServerUtils.runElasticsearchTests(); + + Archives.stopElasticsearch(installation); + } + + @Test + public void test60AutoCreateKeystore() { + assumeThat(installation, is(notNullValue())); + + assertThat(installation.config("elasticsearch.keystore"), file(File, ARCHIVE_OWNER, ARCHIVE_OWNER, p660)); + + final Shell sh = new Shell(); + final Result result; + if (Platforms.WINDOWS) { + result = sh.powershell(installation.bin("elasticsearch-keystore.bat") + " list"); + } else { + result = sh.bash("sudo -u " + ARCHIVE_OWNER + " " + installation.bin("elasticsearch-keystore") + " list"); + } + assertThat(result.stdout, containsString("keystore.seed")); + } + + @Test + public void test70CustomPathConfAndJvmOptions() { + assumeThat(installation, is(notNullValue())); + + final Path tempConf = getTempDir().resolve("esconf-alternate"); + + try { + mkdir(tempConf); + cp(installation.config("elasticsearch.yml"), tempConf.resolve("elasticsearch.yml")); + cp(installation.config("log4j2.properties"), tempConf.resolve("log4j2.properties")); + + // we have to disable Log4j from using JMX lest it will hit a security + // manager exception before we have configured logging; this will fail + // startup since we detect usages of logging before it is configured + final String jvmOptions = + "-Xms512m\n" + + "-Xmx512m\n" + + "-Dlog4j2.disable.jmx=true\n"; + append(tempConf.resolve("jvm.options"), jvmOptions); + + final Shell sh = new Shell(); + if (Platforms.WINDOWS) { + sh.powershell( + "$account = New-Object System.Security.Principal.NTAccount 'vagrant'; " + + "$tempConf = Get-ChildItem '" + tempConf + "' -Recurse; " + + "$tempConf += Get-Item '" + tempConf + "'; " + + "$tempConf | ForEach-Object { " + + "$acl = Get-Acl $_.FullName; " + + "$acl.SetOwner($account); " + + "Set-Acl $_.FullName $acl " + + "}" + ); + } else { + sh.bash("chown -R elasticsearch:elasticsearch " + tempConf); + } + + final Shell serverShell = new Shell(); + serverShell.getEnv().put("ES_PATH_CONF", tempConf.toString()); + serverShell.getEnv().put("ES_JAVA_OPTS", "-XX:-UseCompressedOops"); + + Archives.runElasticsearch(installation, serverShell); + + final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes")); + assertThat(nodesResponse, containsString("\"heap_init_in_bytes\":536870912")); + assertThat(nodesResponse, containsString("\"using_compressed_ordinary_object_pointers\":\"false\"")); + + Archives.stopElasticsearch(installation); + + } finally { + rm(tempConf); + } + } + + @Test + public void test80RelativePathConf() { + assumeThat(installation, is(notNullValue())); + + final Path temp = getTempDir().resolve("esconf-alternate"); + final Path tempConf = temp.resolve("config"); + + try { + mkdir(tempConf); + Stream.of( + "elasticsearch.yml", + "log4j2.properties", + "jvm.options" + ).forEach(file -> cp(installation.config(file), tempConf.resolve(file))); + + append(tempConf.resolve("elasticsearch.yml"), "node.name: relative"); + + final Shell sh = new Shell(); + if (Platforms.WINDOWS) { + sh.powershell( + "$account = New-Object System.Security.Principal.NTAccount 'vagrant'; " + + "$tempConf = Get-ChildItem '" + temp + "' -Recurse; " + + "$tempConf += Get-Item '" + temp + "'; " + + "$tempConf | ForEach-Object { " + + "$acl = Get-Acl $_.FullName; " + + "$acl.SetOwner($account); " + + "Set-Acl $_.FullName $acl " + + "}" + ); + } else { + sh.bash("chown -R elasticsearch:elasticsearch " + temp); + } + + final Shell serverShell = new Shell(temp); + serverShell.getEnv().put("ES_PATH_CONF", "config"); + Archives.runElasticsearch(installation, serverShell); + + final String nodesResponse = makeRequest(Request.Get("http://localhost:9200/_nodes")); + assertThat(nodesResponse, containsString("\"name\":\"relative\"")); + + Archives.stopElasticsearch(installation); + + } finally { + rm(tempConf); + } + } + + } diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java index c4d3655a55398..ff9c853c821ca 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java @@ -21,9 +21,11 @@ import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.util.List; import java.util.stream.Stream; +import static java.util.stream.Collectors.joining; import static org.elasticsearch.packaging.util.FileMatcher.Fileness.Directory; import static org.elasticsearch.packaging.util.FileMatcher.Fileness.File; import static org.elasticsearch.packaging.util.FileMatcher.file; @@ -36,17 +38,26 @@ import static org.elasticsearch.packaging.util.FileUtils.lsGlob; import static org.elasticsearch.packaging.util.FileUtils.mv; +import static org.elasticsearch.packaging.util.FileUtils.slurp; import static org.elasticsearch.packaging.util.Platforms.isDPKG; import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.isEmptyOrNullString; import static org.hamcrest.core.Is.is; import static org.hamcrest.collection.IsEmptyCollection.empty; import static org.hamcrest.collection.IsCollectionWithSize.hasSize; +import static org.hamcrest.core.IsNot.not; +import static org.junit.Assert.assertTrue; /** * Installation and verification logic for archive distributions */ public class Archives { + // in the future we'll run as a role user on Windows + public static final String ARCHIVE_OWNER = Platforms.WINDOWS + ? "vagrant" + : "elasticsearch"; + public static Installation installArchive(Distribution distribution) { return installArchive(distribution, getDefaultArchiveInstallPath(), getCurrentVersion()); } @@ -95,6 +106,8 @@ public static Installation installArchive(Distribution distribution, Path fullIn if (Platforms.LINUX) { setupArchiveUsersLinux(fullInstallPath); + } else { + setupArchiveUsersWindows(fullInstallPath); } return new Installation(fullInstallPath); @@ -134,17 +147,26 @@ private static void setupArchiveUsersLinux(Path installPath) { sh.bash("chown -R elasticsearch:elasticsearch " + installPath); } + private static void setupArchiveUsersWindows(Path installPath) { + // we want the installation to be owned as the vagrant user rather than the Administrators group + + final Shell sh = new Shell(); + sh.powershell( + "$account = New-Object System.Security.Principal.NTAccount 'vagrant'; " + + "$install = Get-ChildItem -Path '" + installPath + "' -Recurse; " + + "$install += Get-Item -Path '" + installPath + "'; " + + "$install | ForEach-Object { " + + "$acl = Get-Acl $_.FullName; " + + "$acl.SetOwner($account); " + + "Set-Acl $_.FullName $acl " + + "}" + ); + } + public static void verifyArchiveInstallation(Installation installation, Distribution distribution) { - // on Windows for now we leave the installation owned by the vagrant user that the tests run as. Since the vagrant account - // is a local administrator, the files really end up being owned by the local administrators group. In the future we'll - // install and run elasticesearch with a role user on Windows - final String owner = Platforms.WINDOWS - ? "BUILTIN\\Administrators" - : "elasticsearch"; - - verifyOssInstallation(installation, distribution, owner); + verifyOssInstallation(installation, distribution, ARCHIVE_OWNER); if (distribution.flavor == Distribution.Flavor.DEFAULT) { - verifyDefaultInstallation(installation, distribution, owner); + verifyDefaultInstallation(installation, distribution, ARCHIVE_OWNER); } } @@ -160,38 +182,38 @@ private static void verifyOssInstallation(Installation es, Distribution distribu assertThat(Files.exists(es.data), is(false)); assertThat(Files.exists(es.scripts), is(false)); - assertThat(es.home.resolve("bin"), file(Directory, owner, owner, p755)); - assertThat(es.home.resolve("lib"), file(Directory, owner, owner, p755)); - assertThat(Files.exists(es.config.resolve("elasticsearch.keystore")), is(false)); + assertThat(es.bin, file(Directory, owner, owner, p755)); + assertThat(es.lib, file(Directory, owner, owner, p755)); + assertThat(Files.exists(es.config("elasticsearch.keystore")), is(false)); Stream.of( - "bin/elasticsearch", - "bin/elasticsearch-env", - "bin/elasticsearch-keystore", - "bin/elasticsearch-plugin", - "bin/elasticsearch-translog" + "elasticsearch", + "elasticsearch-env", + "elasticsearch-keystore", + "elasticsearch-plugin", + "elasticsearch-translog" ).forEach(executable -> { - assertThat(es.home.resolve(executable), file(File, owner, owner, p755)); + assertThat(es.bin(executable), file(File, owner, owner, p755)); if (distribution.packaging == Distribution.Packaging.ZIP) { - assertThat(es.home.resolve(executable + ".bat"), file(File, owner)); + assertThat(es.bin(executable + ".bat"), file(File, owner)); } }); if (distribution.packaging == Distribution.Packaging.ZIP) { Stream.of( - "bin/elasticsearch-service.bat", - "bin/elasticsearch-service-mgr.exe", - "bin/elasticsearch-service-x64.exe" - ).forEach(executable -> assertThat(es.home.resolve(executable), file(File, owner))); + "elasticsearch-service.bat", + "elasticsearch-service-mgr.exe", + "elasticsearch-service-x64.exe" + ).forEach(executable -> assertThat(es.bin(executable), file(File, owner))); } Stream.of( "elasticsearch.yml", "jvm.options", "log4j2.properties" - ).forEach(config -> assertThat(es.config.resolve(config), file(File, owner, owner, p660))); + ).forEach(config -> assertThat(es.config(config), file(File, owner, owner, p660))); Stream.of( "NOTICE.txt", @@ -203,30 +225,30 @@ private static void verifyOssInstallation(Installation es, Distribution distribu private static void verifyDefaultInstallation(Installation es, Distribution distribution, String owner) { Stream.of( - "bin/elasticsearch-certgen", - "bin/elasticsearch-certutil", - "bin/elasticsearch-croneval", - "bin/elasticsearch-migrate", - "bin/elasticsearch-saml-metadata", - "bin/elasticsearch-setup-passwords", - "bin/elasticsearch-sql-cli", - "bin/elasticsearch-syskeygen", - "bin/elasticsearch-users", - "bin/x-pack-env", - "bin/x-pack-security-env", - "bin/x-pack-watcher-env" + "elasticsearch-certgen", + "elasticsearch-certutil", + "elasticsearch-croneval", + "elasticsearch-migrate", + "elasticsearch-saml-metadata", + "elasticsearch-setup-passwords", + "elasticsearch-sql-cli", + "elasticsearch-syskeygen", + "elasticsearch-users", + "x-pack-env", + "x-pack-security-env", + "x-pack-watcher-env" ).forEach(executable -> { - assertThat(es.home.resolve(executable), file(File, owner, owner, p755)); + assertThat(es.bin(executable), file(File, owner, owner, p755)); if (distribution.packaging == Distribution.Packaging.ZIP) { - assertThat(es.home.resolve(executable + ".bat"), file(File, owner)); + assertThat(es.bin(executable + ".bat"), file(File, owner)); } }); // at this time we only install the current version of archive distributions, but if that changes we'll need to pass // the version through here - assertThat(es.home.resolve("bin/elasticsearch-sql-cli-" + getCurrentVersion() + ".jar"), file(File, owner, owner, p755)); + assertThat(es.bin("elasticsearch-sql-cli-" + getCurrentVersion() + ".jar"), file(File, owner, owner, p755)); Stream.of( "users", @@ -234,7 +256,76 @@ private static void verifyDefaultInstallation(Installation es, Distribution dist "roles.yml", "role_mapping.yml", "log4j2.properties" - ).forEach(config -> assertThat(es.config.resolve(config), file(File, owner, owner, p660))); + ).forEach(config -> assertThat(es.config(config), file(File, owner, owner, p660))); + } + + public static void runElasticsearch(Installation installation) { + runElasticsearch(installation, new Shell()); + } + + public static void runElasticsearch(Installation installation, Shell sh) { + final Path pidFile = installation.home.resolve("elasticsearch.pid"); + + if (Platforms.LINUX) { + + // If jayatana is installed then we try to use it. Elasticsearch should ignore it even when we try. + // If it doesn't ignore it then Elasticsearch will fail to start because of security errors. + // This line is attempting to emulate the on login behavior of /usr/share/upstart/sessions/jayatana.conf + if (Files.exists(Paths.get("/usr/share/java/jayatanaag.jar"))) { + sh.getEnv().put("JAVA_TOOL_OPTIONS", "-javaagent:/usr/share/java/jayatanaag.jar"); + } + sh.bash("sudo -E -u " + ARCHIVE_OWNER + " " + + installation.bin("elasticsearch") + " -d -p " + installation.home.resolve("elasticsearch.pid")); + } else { + + // this starts the server in the background. the -d flag is unsupported on windows + // these tests run as Administrator. we don't want to run the server as Administrator, so we provide the current user's + // username and password to the process which has the effect of starting it not as Administrator. + sh.powershell( + "$password = ConvertTo-SecureString 'vagrant' -AsPlainText -Force; " + + "$processInfo = New-Object System.Diagnostics.ProcessStartInfo; " + + "$processInfo.FileName = '" + installation.bin("elasticsearch.bat") + "'; " + + "$processInfo.Arguments = '-p " + installation.home.resolve("elasticsearch.pid") + "'; " + + "$processInfo.Username = 'vagrant'; " + + "$processInfo.Password = $password; " + + "$processInfo.RedirectStandardOutput = $true; " + + "$processInfo.RedirectStandardError = $true; " + + sh.env.entrySet().stream() + .map(entry -> "$processInfo.Environment.Add('" + entry.getKey() + "', '" + entry.getValue() + "'); ") + .collect(joining()) + + "$processInfo.UseShellExecute = $false; " + + "$process = New-Object System.Diagnostics.Process; " + + "$process.StartInfo = $processInfo; " + + "$process.Start() | Out-Null; " + + "$process.Id;" + ); + } + + ServerUtils.waitForElasticsearch(); + + assertTrue(Files.exists(pidFile)); + String pid = slurp(pidFile).trim(); + assertThat(pid, not(isEmptyOrNullString())); + + if (Platforms.LINUX) { + sh.bash("ps " + pid); + } else { + sh.powershell("Get-Process -Id " + pid); + } + } + + public static void stopElasticsearch(Installation installation) { + Path pidFile = installation.home.resolve("elasticsearch.pid"); + assertTrue(Files.exists(pidFile)); + String pid = slurp(pidFile).trim(); + assertThat(pid, not(isEmptyOrNullString())); + + final Shell sh = new Shell(); + if (Platforms.LINUX) { + sh.bash("kill -SIGTERM " + pid); + } else { + sh.powershell("Get-Process -Id " + pid + " | Stop-Process -Force"); + } } } diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/FileUtils.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/FileUtils.java index ad826675244a0..315dc6ffee1f9 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/FileUtils.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/FileUtils.java @@ -21,11 +21,14 @@ import org.elasticsearch.core.internal.io.IOUtils; +import java.io.BufferedWriter; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.StandardOpenOption; import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.attribute.FileOwnerAttributeView; import java.nio.file.attribute.PosixFileAttributes; @@ -63,6 +66,22 @@ public static void rm(Path... paths) { } } + public static Path mkdir(Path path) { + try { + return Files.createDirectories(path); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public static Path cp(Path source, Path target) { + try { + return Files.copy(source, target); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + public static Path mv(Path source, Path target) { try { return Files.move(source, target); @@ -71,9 +90,19 @@ public static Path mv(Path source, Path target) { } } + public static void append(Path file, String text) { + try (BufferedWriter writer = Files.newBufferedWriter(file, StandardCharsets.UTF_8, + StandardOpenOption.CREATE, StandardOpenOption.APPEND)) { + + writer.write(text); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + public static String slurp(Path file) { try { - return String.join("\n", Files.readAllLines(file)); + return String.join("\n", Files.readAllLines(file, StandardCharsets.UTF_8)); } catch (IOException e) { throw new RuntimeException(e); } diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.java index d231762d06227..81223a2424aa6 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.java @@ -27,6 +27,8 @@ public class Installation { public final Path home; + public final Path bin; // this isn't a first-class installation feature but we include it for convenience + public final Path lib; // same public final Path config; public final Path data; public final Path logs; @@ -36,6 +38,9 @@ public class Installation { public Installation(Path home, Path config, Path data, Path logs, Path plugins, Path modules, Path scripts) { this.home = home; + this.bin = home.resolve("bin"); + this.lib = home.resolve("lib"); + this.config = config; this.data = data; this.logs = logs; @@ -55,4 +60,12 @@ public Installation(Path home) { home.resolve("scripts") ); } + + public Path bin(String executableName) { + return bin.resolve(executableName); + } + + public Path config(String configFileName) { + return config.resolve(configFileName); + } } diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/ServerUtils.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/ServerUtils.java new file mode 100644 index 0000000000000..3e8439b68ae60 --- /dev/null +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/ServerUtils.java @@ -0,0 +1,125 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.packaging.util; + +import org.apache.http.HttpResponse; +import org.apache.http.client.fluent.Request; +import org.apache.http.conn.HttpHostConnectException; +import org.apache.http.entity.ContentType; +import org.apache.http.util.EntityUtils; + +import java.io.IOException; +import java.util.Objects; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.containsString; + +public class ServerUtils { + + public static void waitForElasticsearch() { + waitForElasticsearch("green", null); + } + + public static void waitForElasticsearch(String status, String index) { + + Objects.requireNonNull(status); + + // httpclient does not appear to support waiting between retries when the connection is refused, so we loop here + boolean started = false; + for (int i = 0; i < 120; i++) { + try { + + final HttpResponse response = Request.Get("http://localhost:9200/_cluster/health") + .connectTimeout(120000) + .socketTimeout(120000) + .execute() + .returnResponse(); + + if (response.getStatusLine().getStatusCode() >= 300) { + final String statusLine = response.getStatusLine().toString(); + final String body = EntityUtils.toString(response.getEntity()); + throw new RuntimeException("Connecting to elasticsearch cluster health API failed:\n" + statusLine+ "\n" + body); + } + + started = true; + break; + + } catch (HttpHostConnectException e) { + // we want to retry if the connection is refused, so do nothing here + } catch (IOException e) { + throw new RuntimeException(e); + } + + try { + Thread.sleep(1000); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + + } + + if (started == false) { + throw new RuntimeException("Elasticsearch did not start"); + } + + final String url; + if (index == null) { + url = "http://localhost:9200/_cluster/health?wait_for_status=" + status + "&timeout=60s&pretty"; + } else { + url = "http://localhost:9200/_cluster/health/" + index + "?wait_for_status=" + status + "&timeout=60s&pretty"; + + } + + final String body = makeRequest(Request.Get(url)); + assertThat("cluster health response must contain desired status", body, containsString(status)); + } + + public static void runElasticsearchTests() { + makeRequest( + Request.Post("http://localhost:9200/library/book/1?refresh=true&pretty") + .bodyString("{ \"title\": \"Book #1\", \"pages\": 123 }", ContentType.APPLICATION_JSON)); + + makeRequest( + Request.Post("http://localhost:9200/library/book/2?refresh=true&pretty") + .bodyString("{ \"title\": \"Book #2\", \"pages\": 456 }", ContentType.APPLICATION_JSON)); + + String count = makeRequest(Request.Get("http://localhost:9200/_count?pretty")); + assertThat(count, containsString("\"count\" : 2")); + + makeRequest(Request.Delete("http://localhost:9200/_all")); + } + + public static String makeRequest(Request request) { + try { + final HttpResponse response = request.execute().returnResponse(); + + final String body = EntityUtils.toString(response.getEntity()); + + if (response.getStatusLine().getStatusCode() >= 300) { + throw new RuntimeException("Request failed:\n" + response.getStatusLine().toString() + "\n" + body); + } + + return body; + } catch (IOException e) { + throw new RuntimeException("Request failed", e); + } + + } +} diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Shell.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Shell.java index 9a908e2d680eb..a66a667b5ad76 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Shell.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Shell.java @@ -58,6 +58,10 @@ public Shell(Map env, Path workingDirectory) { this.workingDirectory = workingDirectory; } + public Map getEnv() { + return env; + } + /** * Runs a script in a bash shell, throwing an exception if its exit code is nonzero */ diff --git a/qa/vagrant/src/test/resources/packaging/tests/20_tar_package.bats b/qa/vagrant/src/test/resources/packaging/tests/20_tar_package.bats deleted file mode 100644 index 3607e4ab45ac6..0000000000000 --- a/qa/vagrant/src/test/resources/packaging/tests/20_tar_package.bats +++ /dev/null @@ -1,178 +0,0 @@ -#!/usr/bin/env bats - -# This file is used to test the tar gz package. - -# WARNING: This testing file must be executed as root and can -# dramatically change your system. It should only be executed -# in a throw-away VM like those made by the Vagrantfile at -# the root of the Elasticsearch source code. This should -# cause the script to fail if it is executed any other way: -[ -f /etc/is_vagrant_vm ] || { - >&2 echo "must be run on a vagrant VM" - exit 1 -} - -# The test case can be executed with the Bash Automated -# Testing System tool available at https://github.com/sstephenson/bats -# Thanks to Sam Stephenson! - -# Licensed to Elasticsearch under one or more contributor -# license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright -# ownership. Elasticsearch licenses this file to you under -# the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# Load test utilities -load $BATS_UTILS/utils.bash -load $BATS_UTILS/tar.bash -load $BATS_UTILS/plugins.bash - -setup() { - skip_not_tar_gz - export ESHOME=/tmp/elasticsearch - export_elasticsearch_paths -} - -################################## -# Install TAR GZ package -################################## -@test "[TAR] tar command is available" { - # Cleans everything for the 1st execution - clean_before_test - run tar --version - [ "$status" -eq 0 ] -} - -@test "[TAR] archive is available" { - local version=$(cat version) - count=$(find . -type f -name "${PACKAGE_NAME}-${version}.tar.gz" | wc -l) - [ "$count" -eq 1 ] -} - -@test "[TAR] archive is not installed" { - count=$(find /tmp -type d -name 'elasticsearch*' | wc -l) - [ "$count" -eq 0 ] -} - -@test "[TAR] install archive" { - # Install the archive - install_archive - set_debug_logging - - count=$(find /tmp -type d -name 'elasticsearch*' | wc -l) - [ "$count" -eq 1 ] - - # Its simpler to check that the install was correct in this test rather - # than in another test because install_archive sets a number of path - # variables that verify_archive_installation reads. To separate this into - # another test you'd have to recreate the variables. - verify_archive_installation -} - -@test "[TAR] verify elasticsearch-plugin list runs without any plugins installed" { - # previously this would fail because the archive installations did - # not create an empty plugins directory - local plugins_list=`$ESHOME/bin/elasticsearch-plugin list` - [[ -z $plugins_list ]] -} - -@test "[TAR] elasticsearch fails if java executable is not found" { - local JAVA=$(which java) - - sudo chmod -x $JAVA - run "$ESHOME/bin/elasticsearch" - sudo chmod +x $JAVA - - [ "$status" -eq 1 ] - local expected="could not find java; set JAVA_HOME or ensure java is in PATH" - [[ "$output" == *"$expected"* ]] || { - echo "Expected error message [$expected] but found: $output" - false - } -} - -@test "[TAR] test creating elasticearch.keystore" { - sudo -E -u elasticsearch "$ESHOME/bin/elasticsearch-keystore" create - assert_file "$ESCONFIG/elasticsearch.keystore" f elasticsearch elasticsearch 660 - sudo -E -u elasticsearch "$ESHOME/bin/elasticsearch-keystore" list | grep "keystore.seed" - # cleanup for the next test - rm -rf "$ESCONFIG/elasticsearch.keystore" -} - -################################## -# Check that Elasticsearch is working -################################## -@test "[TAR] test elasticsearch" { - start_elasticsearch_service - run_elasticsearch_tests - stop_elasticsearch_service -} - -@test "[TAR] test auto-creating elasticearch.keystore" { - # a keystore should automatically be created after the service is started - assert_file "$ESCONFIG/elasticsearch.keystore" f elasticsearch elasticsearch 660 - # the keystore should be seeded - sudo -E -u elasticsearch "$ESHOME/bin/elasticsearch-keystore" list | grep "keystore.seed" -} - -@test "[TAR] start Elasticsearch with custom JVM options" { - local es_java_opts=$ES_JAVA_OPTS - local es_path_conf=$ES_PATH_CONF - local temp=`mktemp -d` - cp "$ESCONFIG"/elasticsearch.yml "$temp" - cp "$ESCONFIG"/log4j2.properties "$temp" - touch "$temp/jvm.options" - chown -R elasticsearch:elasticsearch "$temp" - echo "-Xms512m" >> "$temp/jvm.options" - echo "-Xmx512m" >> "$temp/jvm.options" - # we have to disable Log4j from using JMX lest it will hit a security - # manager exception before we have configured logging; this will fail - # startup since we detect usages of logging before it is configured - echo "-Dlog4j2.disable.jmx=true" >> "$temp/jvm.options" - export ES_PATH_CONF="$temp" - export ES_JAVA_OPTS="-XX:-UseCompressedOops" - start_elasticsearch_service - curl -s -XGET localhost:9200/_nodes | fgrep '"heap_init_in_bytes":536870912' - curl -s -XGET localhost:9200/_nodes | fgrep '"using_compressed_ordinary_object_pointers":"false"' - stop_elasticsearch_service - export ES_PATH_CONF=$es_path_conf - export ES_JAVA_OPTS=$es_java_opts -} - -@test "[TAR] GC logs exist" { - start_elasticsearch_service - assert_file_exist $ESHOME/logs/gc.log.0.current - stop_elasticsearch_service -} - -@test "[TAR] relative ES_PATH_CONF" { - local es_path_conf=$ES_PATH_CONF - local temp=`mktemp -d` - mkdir "$temp"/config - cp "$ESCONFIG"/elasticsearch.yml "$temp"/config - cp "$ESCONFIG"/log4j2.properties "$temp"/config - cp "$ESCONFIG/jvm.options" "$temp/config" - chown -R elasticsearch:elasticsearch "$temp" - echo "node.name: relative" >> "$temp"/config/elasticsearch.yml - cd "$temp" - export ES_PATH_CONF=config - start_elasticsearch_service - curl -s -XGET localhost:9200/_nodes | fgrep '"name":"relative"' - stop_elasticsearch_service - export ES_PATH_CONF=$es_path_conf -} - -@test "[TAR] remove tar" { - rm -rf "/tmp/elasticsearch" -} From 17291fa0b0fbcf6251de6ef4b45035450a0bdd97 Mon Sep 17 00:00:00 2001 From: Andy Bristol Date: Tue, 3 Jul 2018 13:17:12 -0700 Subject: [PATCH 2/5] review: no delay in waiting for server --- .../packaging/test/ArchiveTestCase.java | 7 ++- .../packaging/util/Archives.java | 5 +- .../packaging/util/ServerUtils.java | 55 +++++++++---------- 3 files changed, 33 insertions(+), 34 deletions(-) diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java index bff5b3f9aece7..48e6be9d129a2 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java @@ -34,6 +34,7 @@ import org.elasticsearch.packaging.util.Distribution; import org.elasticsearch.packaging.util.Installation; +import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.Arrays; @@ -175,7 +176,7 @@ public void test40CreateKeystoreManually() { } @Test - public void test50StartAndStop() { + public void test50StartAndStop() throws IOException { assumeThat(installation, is(notNullValue())); Archives.runElasticsearch(installation); @@ -206,7 +207,7 @@ public void test60AutoCreateKeystore() { } @Test - public void test70CustomPathConfAndJvmOptions() { + public void test70CustomPathConfAndJvmOptions() throws IOException { assumeThat(installation, is(notNullValue())); final Path tempConf = getTempDir().resolve("esconf-alternate"); @@ -259,7 +260,7 @@ public void test70CustomPathConfAndJvmOptions() { } @Test - public void test80RelativePathConf() { + public void test80RelativePathConf() throws IOException { assumeThat(installation, is(notNullValue())); final Path temp = getTempDir().resolve("esconf-alternate"); diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java index ff9c853c821ca..31554b793e5f4 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java @@ -19,6 +19,7 @@ package org.elasticsearch.packaging.util; +import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -259,11 +260,11 @@ private static void verifyDefaultInstallation(Installation es, Distribution dist ).forEach(config -> assertThat(es.config(config), file(File, owner, owner, p660))); } - public static void runElasticsearch(Installation installation) { + public static void runElasticsearch(Installation installation) throws IOException { runElasticsearch(installation, new Shell()); } - public static void runElasticsearch(Installation installation, Shell sh) { + public static void runElasticsearch(Installation installation, Shell sh) throws IOException { final Path pidFile = installation.home.resolve("elasticsearch.pid"); if (Platforms.LINUX) { diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/ServerUtils.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/ServerUtils.java index 3e8439b68ae60..0fc9ddb57849d 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/ServerUtils.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/ServerUtils.java @@ -19,6 +19,8 @@ package org.elasticsearch.packaging.util; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.http.HttpResponse; import org.apache.http.client.fluent.Request; import org.apache.http.conn.HttpHostConnectException; @@ -27,28 +29,35 @@ import java.io.IOException; import java.util.Objects; +import java.util.concurrent.TimeUnit; import static org.hamcrest.MatcherAssert.assertThat; import static org.hamcrest.Matchers.containsString; public class ServerUtils { - public static void waitForElasticsearch() { + private static final Log LOG = LogFactory.getLog(ServerUtils.class); + + private static final long waitTime = TimeUnit.SECONDS.toMillis(60); + + public static void waitForElasticsearch() throws IOException { waitForElasticsearch("green", null); } - public static void waitForElasticsearch(String status, String index) { + public static void waitForElasticsearch(String status, String index) throws IOException { Objects.requireNonNull(status); - // httpclient does not appear to support waiting between retries when the connection is refused, so we loop here + // we loop here rather than letting httpclient handle retries so we can measure the entire waiting time + final long startTime = System.currentTimeMillis(); + long timeElapsed = 0; boolean started = false; - for (int i = 0; i < 120; i++) { + while (started == false && timeElapsed < waitTime) { try { final HttpResponse response = Request.Get("http://localhost:9200/_cluster/health") - .connectTimeout(120000) - .socketTimeout(120000) + .connectTimeout((int) waitTime) + .socketTimeout((int) waitTime) .execute() .returnResponse(); @@ -59,20 +68,13 @@ public static void waitForElasticsearch(String status, String index) { } started = true; - break; } catch (HttpHostConnectException e) { - // we want to retry if the connection is refused, so do nothing here - } catch (IOException e) { - throw new RuntimeException(e); - } - - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - throw new RuntimeException(e); + // we want to retry if the connection is refused + LOG.info("Got connection refused when waiting for cluster health", e); } + timeElapsed = System.currentTimeMillis() - startTime; } if (started == false) { @@ -91,7 +93,7 @@ public static void waitForElasticsearch(String status, String index) { assertThat("cluster health response must contain desired status", body, containsString(status)); } - public static void runElasticsearchTests() { + public static void runElasticsearchTests() throws IOException { makeRequest( Request.Post("http://localhost:9200/library/book/1?refresh=true&pretty") .bodyString("{ \"title\": \"Book #1\", \"pages\": 123 }", ContentType.APPLICATION_JSON)); @@ -106,20 +108,15 @@ public static void runElasticsearchTests() { makeRequest(Request.Delete("http://localhost:9200/_all")); } - public static String makeRequest(Request request) { - try { - final HttpResponse response = request.execute().returnResponse(); + public static String makeRequest(Request request) throws IOException { + final HttpResponse response = request.execute().returnResponse(); + final String body = EntityUtils.toString(response.getEntity()); - final String body = EntityUtils.toString(response.getEntity()); - - if (response.getStatusLine().getStatusCode() >= 300) { - throw new RuntimeException("Request failed:\n" + response.getStatusLine().toString() + "\n" + body); - } - - return body; - } catch (IOException e) { - throw new RuntimeException("Request failed", e); + if (response.getStatusLine().getStatusCode() >= 300) { + throw new RuntimeException("Request failed:\n" + response.getStatusLine().toString() + "\n" + body); } + return body; + } } From 2686d60d6c4f5d19f95040329fa873112ed69aef Mon Sep 17 00:00:00 2001 From: Andy Bristol Date: Tue, 3 Jul 2018 15:38:30 -0700 Subject: [PATCH 3/5] review: change platform conditionals to helper methods --- .../packaging/test/ArchiveTestCase.java | 96 +++++++++---------- .../packaging/util/Archives.java | 44 ++++----- .../elasticsearch/packaging/util/Cleanup.java | 23 ++--- .../packaging/util/Platforms.java | 12 +++ 4 files changed, 87 insertions(+), 88 deletions(-) diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java index 48e6be9d129a2..80748db2658f1 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java @@ -96,6 +96,7 @@ public void test10Install() { public void test20PluginsListWithNoPlugins() { assumeThat(installation, is(notNullValue())); + // todo here final Shell sh = new Shell(); final Result r = Platforms.WINDOWS ? sh.powershell(installation.bin("elasticsearch-plugin.bat") + " list") @@ -109,7 +110,8 @@ public void test30AbortWhenJavaMissing() { assumeThat(installation, is(notNullValue())); final Shell sh = new Shell(); - if (Platforms.WINDOWS) { + + Platforms.onWindows(() -> { // on windows, removing java from PATH and removing JAVA_HOME is less involved than changing the permissions of the java // executable. we also don't check permissions in the windows scripts anyway final String originalPath = sh.powershell("$Env:PATH").stdout.trim(); @@ -131,8 +133,9 @@ public void test30AbortWhenJavaMissing() { assertThat(runResult.exitCode, is(1)); assertThat(runResult.stderr, containsString("could not find java; set JAVA_HOME or ensure java is in PATH")); + }); - } else { + Platforms.onLinux(() -> { final String javaPath = sh.bash("which java").stdout.trim(); sh.bash("chmod -x '" + javaPath + "'"); final Result runResult = sh.bashIgnoreExitCode(installation.bin("elasticsearch").toString()); @@ -140,7 +143,7 @@ public void test30AbortWhenJavaMissing() { assertThat(runResult.exitCode, is(1)); assertThat(runResult.stdout, containsString("could not find java; set JAVA_HOME or ensure java is in PATH")); - } + }); } @Test @@ -148,24 +151,24 @@ public void test40CreateKeystoreManually() { assumeThat(installation, is(notNullValue())); final Shell sh = new Shell(); - if (Platforms.WINDOWS) { - // this is a hack around the fact that we can't run a command in the same session as the same user but not as administrator. - // the keystore ends up being owned by the Administrators group, so we manually set it to be owned by the vagrant user here. - // from the server's perspective the permissions aren't really different, this is just to reflect what we'd expect in the tests. - // when we run these commands as a role user we won't have to do this - sh.powershell( + + Platforms.onLinux(() -> sh.bash("sudo -u " + ARCHIVE_OWNER + " " + installation.bin("elasticsearch-keystore") + " create")); + + // this is a hack around the fact that we can't run a command in the same session as the same user but not as administrator. + // the keystore ends up being owned by the Administrators group, so we manually set it to be owned by the vagrant user here. + // from the server's perspective the permissions aren't really different, this is just to reflect what we'd expect in the tests. + // when we run these commands as a role user we won't have to do this + Platforms.onWindows(() -> sh.powershell( installation.bin("elasticsearch-keystore.bat") + " create; " + "$account = New-Object System.Security.Principal.NTAccount 'vagrant'; " + "$acl = Get-Acl '" + installation.config("elasticsearch.keystore") + "'; " + "$acl.SetOwner($account); " + "Set-Acl '" + installation.config("elasticsearch.keystore") + "' $acl" - ); - } else { - sh.bash("sudo -u " + ARCHIVE_OWNER + " " + installation.bin("elasticsearch-keystore") + " create"); - } + )); assertThat(installation.config("elasticsearch.keystore"), file(File, ARCHIVE_OWNER, ARCHIVE_OWNER, p660)); + // todo here final Result r = Platforms.WINDOWS ? sh.powershell(installation.bin("elasticsearch-keystore.bat") + " list") : sh.bash("sudo -u " + ARCHIVE_OWNER + " " + installation.bin("elasticsearch-keystore") + " list"); @@ -197,13 +200,16 @@ public void test60AutoCreateKeystore() { assertThat(installation.config("elasticsearch.keystore"), file(File, ARCHIVE_OWNER, ARCHIVE_OWNER, p660)); final Shell sh = new Shell(); - final Result result; - if (Platforms.WINDOWS) { - result = sh.powershell(installation.bin("elasticsearch-keystore.bat") + " list"); - } else { - result = sh.bash("sudo -u " + ARCHIVE_OWNER + " " + installation.bin("elasticsearch-keystore") + " list"); - } - assertThat(result.stdout, containsString("keystore.seed")); + + Platforms.onLinux(() -> { + final Result result = sh.bash("sudo -u " + ARCHIVE_OWNER + " " + installation.bin("elasticsearch-keystore") + " list"); + assertThat(result.stdout, containsString("keystore.seed")); + }); + + Platforms.onWindows(() -> { + final Result result = sh.powershell(installation.bin("elasticsearch-keystore.bat") + " list"); + assertThat(result.stdout, containsString("keystore.seed")); + }); } @Test @@ -227,20 +233,17 @@ public void test70CustomPathConfAndJvmOptions() throws IOException { append(tempConf.resolve("jvm.options"), jvmOptions); final Shell sh = new Shell(); - if (Platforms.WINDOWS) { - sh.powershell( - "$account = New-Object System.Security.Principal.NTAccount 'vagrant'; " + - "$tempConf = Get-ChildItem '" + tempConf + "' -Recurse; " + - "$tempConf += Get-Item '" + tempConf + "'; " + - "$tempConf | ForEach-Object { " + - "$acl = Get-Acl $_.FullName; " + - "$acl.SetOwner($account); " + - "Set-Acl $_.FullName $acl " + - "}" - ); - } else { - sh.bash("chown -R elasticsearch:elasticsearch " + tempConf); - } + Platforms.onLinux(() -> sh.bash("chown -R elasticsearch:elasticsearch " + tempConf)); + Platforms.onWindows(() -> sh.powershell( + "$account = New-Object System.Security.Principal.NTAccount 'vagrant'; " + + "$tempConf = Get-ChildItem '" + tempConf + "' -Recurse; " + + "$tempConf += Get-Item '" + tempConf + "'; " + + "$tempConf | ForEach-Object { " + + "$acl = Get-Acl $_.FullName; " + + "$acl.SetOwner($account); " + + "Set-Acl $_.FullName $acl " + + "}" + )); final Shell serverShell = new Shell(); serverShell.getEnv().put("ES_PATH_CONF", tempConf.toString()); @@ -277,20 +280,17 @@ public void test80RelativePathConf() throws IOException { append(tempConf.resolve("elasticsearch.yml"), "node.name: relative"); final Shell sh = new Shell(); - if (Platforms.WINDOWS) { - sh.powershell( - "$account = New-Object System.Security.Principal.NTAccount 'vagrant'; " + - "$tempConf = Get-ChildItem '" + temp + "' -Recurse; " + - "$tempConf += Get-Item '" + temp + "'; " + - "$tempConf | ForEach-Object { " + - "$acl = Get-Acl $_.FullName; " + - "$acl.SetOwner($account); " + - "Set-Acl $_.FullName $acl " + - "}" - ); - } else { - sh.bash("chown -R elasticsearch:elasticsearch " + temp); - } + Platforms.onLinux(() -> sh.bash("chown -R elasticsearch:elasticsearch " + temp)); + Platforms.onWindows(() -> sh.powershell( + "$account = New-Object System.Security.Principal.NTAccount 'vagrant'; " + + "$tempConf = Get-ChildItem '" + temp + "' -Recurse; " + + "$tempConf += Get-Item '" + temp + "'; " + + "$tempConf | ForEach-Object { " + + "$acl = Get-Acl $_.FullName; " + + "$acl.SetOwner($account); " + + "Set-Acl $_.FullName $acl " + + "}" + )); final Shell serverShell = new Shell(temp); serverShell.getEnv().put("ES_PATH_CONF", "config"); diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java index 31554b793e5f4..7732b076fed1e 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java @@ -75,22 +75,21 @@ public static Installation installArchive(Distribution distribution, Path fullIn if (distribution.packaging == Distribution.Packaging.TAR) { - if (Platforms.LINUX) { - sh.bash("tar -C " + baseInstallPath + " -xzpf " + distributionFile); - } else { + Platforms.onLinux(() -> sh.bash("tar -C " + baseInstallPath + " -xzpf " + distributionFile)); + + if (Platforms.WINDOWS) { throw new RuntimeException("Distribution " + distribution + " is not supported on windows"); } } else if (distribution.packaging == Distribution.Packaging.ZIP) { - if (Platforms.LINUX) { - sh.bash("unzip " + distributionFile + " -d " + baseInstallPath); - } else { + Platforms.onLinux(() -> sh.bash("unzip " + distributionFile + " -d " + baseInstallPath)); + + Platforms.onWindows(() -> sh.powershell( "Add-Type -AssemblyName 'System.IO.Compression.Filesystem'; " + "[IO.Compression.ZipFile]::ExtractToDirectory('" + distributionFile + "', '" + baseInstallPath + "')" - ); - } + )); } else { throw new RuntimeException("Distribution " + distribution + " is not a known archive type"); @@ -105,11 +104,8 @@ public static Installation installArchive(Distribution distribution, Path fullIn assertThat("only the intended installation exists", installations, hasSize(1)); assertThat("only the intended installation exists", installations.get(0), is(fullInstallPath)); - if (Platforms.LINUX) { - setupArchiveUsersLinux(fullInstallPath); - } else { - setupArchiveUsersWindows(fullInstallPath); - } + Platforms.onLinux(() -> setupArchiveUsersLinux(fullInstallPath)); + Platforms.onWindows(() -> setupArchiveUsersWindows(fullInstallPath)); return new Installation(fullInstallPath); } @@ -267,8 +263,7 @@ public static void runElasticsearch(Installation installation) throws IOExceptio public static void runElasticsearch(Installation installation, Shell sh) throws IOException { final Path pidFile = installation.home.resolve("elasticsearch.pid"); - if (Platforms.LINUX) { - + Platforms.onLinux(() -> { // If jayatana is installed then we try to use it. Elasticsearch should ignore it even when we try. // If it doesn't ignore it then Elasticsearch will fail to start because of security errors. // This line is attempting to emulate the on login behavior of /usr/share/upstart/sessions/jayatana.conf @@ -277,8 +272,9 @@ public static void runElasticsearch(Installation installation, Shell sh) throws } sh.bash("sudo -E -u " + ARCHIVE_OWNER + " " + installation.bin("elasticsearch") + " -d -p " + installation.home.resolve("elasticsearch.pid")); - } else { + }); + Platforms.onWindows(() -> { // this starts the server in the background. the -d flag is unsupported on windows // these tests run as Administrator. we don't want to run the server as Administrator, so we provide the current user's // username and password to the process which has the effect of starting it not as Administrator. @@ -300,7 +296,7 @@ public static void runElasticsearch(Installation installation, Shell sh) throws "$process.Start() | Out-Null; " + "$process.Id;" ); - } + }); ServerUtils.waitForElasticsearch(); @@ -308,11 +304,8 @@ public static void runElasticsearch(Installation installation, Shell sh) throws String pid = slurp(pidFile).trim(); assertThat(pid, not(isEmptyOrNullString())); - if (Platforms.LINUX) { - sh.bash("ps " + pid); - } else { - sh.powershell("Get-Process -Id " + pid); - } + Platforms.onLinux(() -> sh.bash("ps " + pid)); + Platforms.onWindows(() -> sh.powershell("Get-Process -Id " + pid)); } public static void stopElasticsearch(Installation installation) { @@ -322,11 +315,8 @@ public static void stopElasticsearch(Installation installation) { assertThat(pid, not(isEmptyOrNullString())); final Shell sh = new Shell(); - if (Platforms.LINUX) { - sh.bash("kill -SIGTERM " + pid); - } else { - sh.powershell("Get-Process -Id " + pid + " | Stop-Process -Force"); - } + Platforms.onLinux(() -> sh.bash("kill -SIGTERM " + pid)); + Platforms.onWindows(() -> sh.powershell("Get-Process -Id " + pid + " | Stop-Process -Force")); } } diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Cleanup.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Cleanup.java index a775a23a19490..6968221c65237 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Cleanup.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Cleanup.java @@ -56,31 +56,28 @@ public static void cleanEverything() { final Shell sh = new Shell(); // kill elasticsearch processes - if (Platforms.WINDOWS) { + Platforms.onLinux(() -> { + sh.bashIgnoreExitCode("pkill -u elasticsearch"); + sh.bashIgnoreExitCode("ps aux | grep -i 'org.elasticsearch.bootstrap.Elasticsearch' | awk {'print $2'} | xargs kill -9"); + }); + Platforms.onWindows(() -> { // the view of processes returned by Get-Process doesn't expose command line arguments, so we use WMI here sh.powershellIgnoreExitCode( "Get-WmiObject Win32_Process | " + "Where-Object { $_.CommandLine -Match 'org.elasticsearch.bootstrap.Elasticsearch' } | " + "ForEach-Object { $_.Terminate() }" ); + }); - } else { - - sh.bashIgnoreExitCode("pkill -u elasticsearch"); - sh.bashIgnoreExitCode("ps aux | grep -i 'org.elasticsearch.bootstrap.Elasticsearch' | awk {'print $2'} | xargs kill -9"); - - } - - if (Platforms.LINUX) { - purgePackagesLinux(); - } + Platforms.onLinux(Cleanup::purgePackagesLinux); // remove elasticsearch users - if (Platforms.LINUX) { + Platforms.onLinux(() -> { sh.bashIgnoreExitCode("userdel elasticsearch"); sh.bashIgnoreExitCode("groupdel elasticsearch"); - } + }); + // when we run es as a role user on windows, add the equivalent here // delete files that may still exist lsGlob(getTempDir(), "elasticsearch*").forEach(FileUtils::rm); diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Platforms.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Platforms.java index 202c025ae8a55..5ee9167919e04 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Platforms.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Platforms.java @@ -65,4 +65,16 @@ public static boolean isSysVInit() { } return new Shell().bashIgnoreExitCode("which service").isSuccess(); } + + public static void onWindows(Runnable action) { + if (WINDOWS) { + action.run(); + } + } + + public static void onLinux(Runnable action) { + if (LINUX) { + action.run(); + } + } } From 45dff8af23f77fe8e9c700db988a38ba8bca7d15 Mon Sep 17 00:00:00 2001 From: Andy Bristol Date: Tue, 3 Jul 2018 17:32:26 -0700 Subject: [PATCH 4/5] review: abstract which executables are chosen by platform --- .../packaging/test/ArchiveTestCase.java | 57 ++++++++++--------- .../packaging/util/Archives.java | 45 ++++++++------- .../elasticsearch/packaging/util/Cleanup.java | 20 +++---- .../packaging/util/Installation.java | 19 +++++++ .../packaging/util/Platforms.java | 12 ++-- .../elasticsearch/packaging/util/Shell.java | 48 ++++++---------- 6 files changed, 106 insertions(+), 95 deletions(-) diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java index 80748db2658f1..684df26ab979e 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java @@ -96,11 +96,9 @@ public void test10Install() { public void test20PluginsListWithNoPlugins() { assumeThat(installation, is(notNullValue())); - // todo here + final Installation.Executables bin = installation.executables(); final Shell sh = new Shell(); - final Result r = Platforms.WINDOWS - ? sh.powershell(installation.bin("elasticsearch-plugin.bat") + " list") - : sh.bash(installation.bin("elasticsearch-plugin") + " list"); + final Result r = sh.run(bin.elasticsearchPlugin + " list"); assertThat(r.stdout, isEmptyString()); } @@ -109,26 +107,25 @@ public void test20PluginsListWithNoPlugins() { public void test30AbortWhenJavaMissing() { assumeThat(installation, is(notNullValue())); + final Installation.Executables bin = installation.executables(); final Shell sh = new Shell(); Platforms.onWindows(() -> { // on windows, removing java from PATH and removing JAVA_HOME is less involved than changing the permissions of the java // executable. we also don't check permissions in the windows scripts anyway - final String originalPath = sh.powershell("$Env:PATH").stdout.trim(); + final String originalPath = sh.run("$Env:PATH").stdout.trim(); final String newPath = Arrays.stream(originalPath.split(";")) .filter(path -> path.contains("Java") == false) .collect(joining(";")); - final String javaHome = sh.powershell("$Env:JAVA_HOME").stdout.trim(); - // note the lack of a $ when clearing the JAVA_HOME env variable - with a $ it deletes the java home directory // https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/providers/environment-provider?view=powershell-6 // // this won't persist to another session so we don't have to reset anything - final Result runResult = sh.powershellIgnoreExitCode( + final Result runResult = sh.runIgnoreExitCode( "$Env:PATH = '" + newPath + "'; " + "Remove-Item Env:JAVA_HOME; " + - installation.bin("elasticsearch.bat") + bin.elasticsearch ); assertThat(runResult.exitCode, is(1)); @@ -136,10 +133,10 @@ public void test30AbortWhenJavaMissing() { }); Platforms.onLinux(() -> { - final String javaPath = sh.bash("which java").stdout.trim(); - sh.bash("chmod -x '" + javaPath + "'"); - final Result runResult = sh.bashIgnoreExitCode(installation.bin("elasticsearch").toString()); - sh.bash("chmod +x '" + javaPath + "'"); + final String javaPath = sh.run("which java").stdout.trim(); + sh.run("chmod -x '" + javaPath + "'"); + final Result runResult = sh.runIgnoreExitCode(bin.elasticsearch.toString()); + sh.run("chmod +x '" + javaPath + "'"); assertThat(runResult.exitCode, is(1)); assertThat(runResult.stdout, containsString("could not find java; set JAVA_HOME or ensure java is in PATH")); @@ -150,16 +147,17 @@ public void test30AbortWhenJavaMissing() { public void test40CreateKeystoreManually() { assumeThat(installation, is(notNullValue())); + final Installation.Executables bin = installation.executables(); final Shell sh = new Shell(); - Platforms.onLinux(() -> sh.bash("sudo -u " + ARCHIVE_OWNER + " " + installation.bin("elasticsearch-keystore") + " create")); + Platforms.onLinux(() -> sh.run("sudo -u " + ARCHIVE_OWNER + " " + bin.elasticsearchKeystore + " create")); // this is a hack around the fact that we can't run a command in the same session as the same user but not as administrator. // the keystore ends up being owned by the Administrators group, so we manually set it to be owned by the vagrant user here. // from the server's perspective the permissions aren't really different, this is just to reflect what we'd expect in the tests. // when we run these commands as a role user we won't have to do this - Platforms.onWindows(() -> sh.powershell( - installation.bin("elasticsearch-keystore.bat") + " create; " + + Platforms.onWindows(() -> sh.run( + bin.elasticsearchKeystore + " create; " + "$account = New-Object System.Security.Principal.NTAccount 'vagrant'; " + "$acl = Get-Acl '" + installation.config("elasticsearch.keystore") + "'; " + "$acl.SetOwner($account); " + @@ -168,11 +166,15 @@ public void test40CreateKeystoreManually() { assertThat(installation.config("elasticsearch.keystore"), file(File, ARCHIVE_OWNER, ARCHIVE_OWNER, p660)); - // todo here - final Result r = Platforms.WINDOWS - ? sh.powershell(installation.bin("elasticsearch-keystore.bat") + " list") - : sh.bash("sudo -u " + ARCHIVE_OWNER + " " + installation.bin("elasticsearch-keystore") + " list"); - assertThat(r.stdout, containsString("keystore.seed")); + Platforms.onLinux(() -> { + final Result r = sh.run("sudo -u " + ARCHIVE_OWNER + " " + bin.elasticsearchKeystore + " list"); + assertThat(r.stdout, containsString("keystore.seed")); + }); + + Platforms.onWindows(() -> { + final Result r = sh.run(bin.elasticsearchKeystore + " list"); + assertThat(r.stdout, containsString("keystore.seed")); + }); // cleanup for next test rm(installation.config("elasticsearch.keystore")); @@ -199,15 +201,16 @@ public void test60AutoCreateKeystore() { assertThat(installation.config("elasticsearch.keystore"), file(File, ARCHIVE_OWNER, ARCHIVE_OWNER, p660)); + final Installation.Executables bin = installation.executables(); final Shell sh = new Shell(); Platforms.onLinux(() -> { - final Result result = sh.bash("sudo -u " + ARCHIVE_OWNER + " " + installation.bin("elasticsearch-keystore") + " list"); + final Result result = sh.run("sudo -u " + ARCHIVE_OWNER + " " + bin.elasticsearchKeystore + " list"); assertThat(result.stdout, containsString("keystore.seed")); }); Platforms.onWindows(() -> { - final Result result = sh.powershell(installation.bin("elasticsearch-keystore.bat") + " list"); + final Result result = sh.run(bin.elasticsearchKeystore + " list"); assertThat(result.stdout, containsString("keystore.seed")); }); } @@ -233,8 +236,8 @@ public void test70CustomPathConfAndJvmOptions() throws IOException { append(tempConf.resolve("jvm.options"), jvmOptions); final Shell sh = new Shell(); - Platforms.onLinux(() -> sh.bash("chown -R elasticsearch:elasticsearch " + tempConf)); - Platforms.onWindows(() -> sh.powershell( + Platforms.onLinux(() -> sh.run("chown -R elasticsearch:elasticsearch " + tempConf)); + Platforms.onWindows(() -> sh.run( "$account = New-Object System.Security.Principal.NTAccount 'vagrant'; " + "$tempConf = Get-ChildItem '" + tempConf + "' -Recurse; " + "$tempConf += Get-Item '" + tempConf + "'; " + @@ -280,8 +283,8 @@ public void test80RelativePathConf() throws IOException { append(tempConf.resolve("elasticsearch.yml"), "node.name: relative"); final Shell sh = new Shell(); - Platforms.onLinux(() -> sh.bash("chown -R elasticsearch:elasticsearch " + temp)); - Platforms.onWindows(() -> sh.powershell( + Platforms.onLinux(() -> sh.run("chown -R elasticsearch:elasticsearch " + temp)); + Platforms.onWindows(() -> sh.run( "$account = New-Object System.Security.Principal.NTAccount 'vagrant'; " + "$tempConf = Get-ChildItem '" + temp + "' -Recurse; " + "$tempConf += Get-Item '" + temp + "'; " + diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java index 7732b076fed1e..6ffec813eb041 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Archives.java @@ -75,7 +75,7 @@ public static Installation installArchive(Distribution distribution, Path fullIn if (distribution.packaging == Distribution.Packaging.TAR) { - Platforms.onLinux(() -> sh.bash("tar -C " + baseInstallPath + " -xzpf " + distributionFile)); + Platforms.onLinux(() -> sh.run("tar -C " + baseInstallPath + " -xzpf " + distributionFile)); if (Platforms.WINDOWS) { throw new RuntimeException("Distribution " + distribution + " is not supported on windows"); @@ -83,12 +83,11 @@ public static Installation installArchive(Distribution distribution, Path fullIn } else if (distribution.packaging == Distribution.Packaging.ZIP) { - Platforms.onLinux(() -> sh.bash("unzip " + distributionFile + " -d " + baseInstallPath)); + Platforms.onLinux(() -> sh.run("unzip " + distributionFile + " -d " + baseInstallPath)); - Platforms.onWindows(() -> - sh.powershell( - "Add-Type -AssemblyName 'System.IO.Compression.Filesystem'; " + - "[IO.Compression.ZipFile]::ExtractToDirectory('" + distributionFile + "', '" + baseInstallPath + "')" + Platforms.onWindows(() -> sh.run( + "Add-Type -AssemblyName 'System.IO.Compression.Filesystem'; " + + "[IO.Compression.ZipFile]::ExtractToDirectory('" + distributionFile + "', '" + baseInstallPath + "')" )); } else { @@ -113,17 +112,17 @@ public static Installation installArchive(Distribution distribution, Path fullIn private static void setupArchiveUsersLinux(Path installPath) { final Shell sh = new Shell(); - if (sh.bashIgnoreExitCode("getent group elasticsearch").isSuccess() == false) { + if (sh.runIgnoreExitCode("getent group elasticsearch").isSuccess() == false) { if (isDPKG()) { - sh.bash("addgroup --system elasticsearch"); + sh.run("addgroup --system elasticsearch"); } else { - sh.bash("groupadd -r elasticsearch"); + sh.run("groupadd -r elasticsearch"); } } - if (sh.bashIgnoreExitCode("id elasticsearch").isSuccess() == false) { + if (sh.runIgnoreExitCode("id elasticsearch").isSuccess() == false) { if (isDPKG()) { - sh.bash("adduser " + + sh.run("adduser " + "--quiet " + "--system " + "--no-create-home " + @@ -132,7 +131,7 @@ private static void setupArchiveUsersLinux(Path installPath) { "--shell /bin/false " + "elasticsearch"); } else { - sh.bash("useradd " + + sh.run("useradd " + "--system " + "-M " + "--gid elasticsearch " + @@ -141,14 +140,14 @@ private static void setupArchiveUsersLinux(Path installPath) { "elasticsearch"); } } - sh.bash("chown -R elasticsearch:elasticsearch " + installPath); + sh.run("chown -R elasticsearch:elasticsearch " + installPath); } private static void setupArchiveUsersWindows(Path installPath) { // we want the installation to be owned as the vagrant user rather than the Administrators group final Shell sh = new Shell(); - sh.powershell( + sh.run( "$account = New-Object System.Security.Principal.NTAccount 'vagrant'; " + "$install = Get-ChildItem -Path '" + installPath + "' -Recurse; " + "$install += Get-Item -Path '" + installPath + "'; " + @@ -263,6 +262,8 @@ public static void runElasticsearch(Installation installation) throws IOExceptio public static void runElasticsearch(Installation installation, Shell sh) throws IOException { final Path pidFile = installation.home.resolve("elasticsearch.pid"); + final Installation.Executables bin = installation.executables(); + Platforms.onLinux(() -> { // If jayatana is installed then we try to use it. Elasticsearch should ignore it even when we try. // If it doesn't ignore it then Elasticsearch will fail to start because of security errors. @@ -270,18 +271,18 @@ public static void runElasticsearch(Installation installation, Shell sh) throws if (Files.exists(Paths.get("/usr/share/java/jayatanaag.jar"))) { sh.getEnv().put("JAVA_TOOL_OPTIONS", "-javaagent:/usr/share/java/jayatanaag.jar"); } - sh.bash("sudo -E -u " + ARCHIVE_OWNER + " " + - installation.bin("elasticsearch") + " -d -p " + installation.home.resolve("elasticsearch.pid")); + sh.run("sudo -E -u " + ARCHIVE_OWNER + " " + + bin.elasticsearch + " -d -p " + installation.home.resolve("elasticsearch.pid")); }); Platforms.onWindows(() -> { // this starts the server in the background. the -d flag is unsupported on windows // these tests run as Administrator. we don't want to run the server as Administrator, so we provide the current user's // username and password to the process which has the effect of starting it not as Administrator. - sh.powershell( + sh.run( "$password = ConvertTo-SecureString 'vagrant' -AsPlainText -Force; " + "$processInfo = New-Object System.Diagnostics.ProcessStartInfo; " + - "$processInfo.FileName = '" + installation.bin("elasticsearch.bat") + "'; " + + "$processInfo.FileName = '" + bin.elasticsearch + "'; " + "$processInfo.Arguments = '-p " + installation.home.resolve("elasticsearch.pid") + "'; " + "$processInfo.Username = 'vagrant'; " + "$processInfo.Password = $password; " + @@ -304,8 +305,8 @@ public static void runElasticsearch(Installation installation, Shell sh) throws String pid = slurp(pidFile).trim(); assertThat(pid, not(isEmptyOrNullString())); - Platforms.onLinux(() -> sh.bash("ps " + pid)); - Platforms.onWindows(() -> sh.powershell("Get-Process -Id " + pid)); + Platforms.onLinux(() -> sh.run("ps " + pid)); + Platforms.onWindows(() -> sh.run("Get-Process -Id " + pid)); } public static void stopElasticsearch(Installation installation) { @@ -315,8 +316,8 @@ public static void stopElasticsearch(Installation installation) { assertThat(pid, not(isEmptyOrNullString())); final Shell sh = new Shell(); - Platforms.onLinux(() -> sh.bash("kill -SIGTERM " + pid)); - Platforms.onWindows(() -> sh.powershell("Get-Process -Id " + pid + " | Stop-Process -Force")); + Platforms.onLinux(() -> sh.run("kill -SIGTERM " + pid)); + Platforms.onWindows(() -> sh.run("Get-Process -Id " + pid + " | Stop-Process -Force")); } } diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Cleanup.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Cleanup.java index 6968221c65237..4ff2998988c5f 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Cleanup.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Cleanup.java @@ -57,13 +57,13 @@ public static void cleanEverything() { // kill elasticsearch processes Platforms.onLinux(() -> { - sh.bashIgnoreExitCode("pkill -u elasticsearch"); - sh.bashIgnoreExitCode("ps aux | grep -i 'org.elasticsearch.bootstrap.Elasticsearch' | awk {'print $2'} | xargs kill -9"); + sh.runIgnoreExitCode("pkill -u elasticsearch"); + sh.runIgnoreExitCode("ps aux | grep -i 'org.elasticsearch.bootstrap.Elasticsearch' | awk {'print $2'} | xargs kill -9"); }); Platforms.onWindows(() -> { // the view of processes returned by Get-Process doesn't expose command line arguments, so we use WMI here - sh.powershellIgnoreExitCode( + sh.runIgnoreExitCode( "Get-WmiObject Win32_Process | " + "Where-Object { $_.CommandLine -Match 'org.elasticsearch.bootstrap.Elasticsearch' } | " + "ForEach-Object { $_.Terminate() }" @@ -74,8 +74,8 @@ public static void cleanEverything() { // remove elasticsearch users Platforms.onLinux(() -> { - sh.bashIgnoreExitCode("userdel elasticsearch"); - sh.bashIgnoreExitCode("groupdel elasticsearch"); + sh.runIgnoreExitCode("userdel elasticsearch"); + sh.runIgnoreExitCode("groupdel elasticsearch"); }); // when we run es as a role user on windows, add the equivalent here @@ -92,7 +92,7 @@ public static void cleanEverything() { // disable elasticsearch service // todo add this for windows when adding tests for service intallation if (Platforms.LINUX && isSystemd()) { - sh.bash("systemctl unmask systemd-sysctl.service"); + sh.run("systemctl unmask systemd-sysctl.service"); } } @@ -100,19 +100,19 @@ private static void purgePackagesLinux() { final Shell sh = new Shell(); if (isRPM()) { - sh.bashIgnoreExitCode("rpm --quiet -e elasticsearch elasticsearch-oss"); + sh.runIgnoreExitCode("rpm --quiet -e elasticsearch elasticsearch-oss"); } if (isYUM()) { - sh.bashIgnoreExitCode("yum remove -y elasticsearch elasticsearch-oss"); + sh.runIgnoreExitCode("yum remove -y elasticsearch elasticsearch-oss"); } if (isDPKG()) { - sh.bashIgnoreExitCode("dpkg --purge elasticsearch elasticsearch-oss"); + sh.runIgnoreExitCode("dpkg --purge elasticsearch elasticsearch-oss"); } if (isAptGet()) { - sh.bashIgnoreExitCode("apt-get --quiet --yes purge elasticsearch elasticsearch-oss"); + sh.runIgnoreExitCode("apt-get --quiet --yes purge elasticsearch elasticsearch-oss"); } } } diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.java index 81223a2424aa6..68da440400a36 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Installation.java @@ -68,4 +68,23 @@ public Path bin(String executableName) { public Path config(String configFileName) { return config.resolve(configFileName); } + + public Executables executables() { + return new Executables(); + } + + public class Executables { + + public final Path elasticsearch = platformExecutable("elasticsearch"); + public final Path elasticsearchPlugin = platformExecutable("elasticsearch-plugin"); + public final Path elasticsearchKeystore = platformExecutable("elasticsearch-keystore"); + public final Path elasticsearchTranslog = platformExecutable("elasticsearch-translog"); + + private Path platformExecutable(String name) { + final String platformExecutableName = Platforms.WINDOWS + ? name + ".bat" + : name; + return bin(platformExecutableName); + } + } } diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Platforms.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Platforms.java index 5ee9167919e04..a388231c30382 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Platforms.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Platforms.java @@ -28,42 +28,42 @@ public static boolean isDPKG() { if (WINDOWS) { return false; } - return new Shell().bashIgnoreExitCode("which dpkg").isSuccess(); + return new Shell().runIgnoreExitCode("which dpkg").isSuccess(); } public static boolean isAptGet() { if (WINDOWS) { return false; } - return new Shell().bashIgnoreExitCode("which apt-get").isSuccess(); + return new Shell().runIgnoreExitCode("which apt-get").isSuccess(); } public static boolean isRPM() { if (WINDOWS) { return false; } - return new Shell().bashIgnoreExitCode("which rpm").isSuccess(); + return new Shell().runIgnoreExitCode("which rpm").isSuccess(); } public static boolean isYUM() { if (WINDOWS) { return false; } - return new Shell().bashIgnoreExitCode("which yum").isSuccess(); + return new Shell().runIgnoreExitCode("which yum").isSuccess(); } public static boolean isSystemd() { if (WINDOWS) { return false; } - return new Shell().bashIgnoreExitCode("which systemctl").isSuccess(); + return new Shell().runIgnoreExitCode("which systemctl").isSuccess(); } public static boolean isSysVInit() { if (WINDOWS) { return false; } - return new Shell().bashIgnoreExitCode("which service").isSuccess(); + return new Shell().runIgnoreExitCode("which service").isSuccess(); } public static void onWindows(Runnable action) { diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Shell.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Shell.java index a66a667b5ad76..5853bc2daa148 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Shell.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Shell.java @@ -63,57 +63,45 @@ public Map getEnv() { } /** - * Runs a script in a bash shell, throwing an exception if its exit code is nonzero + * Run the provided string as a shell script. On Linux the {@code bash -c [script]} syntax will be used, and on Windows + * the {@code powershell.exe -Command [script]} syntax will be used. Throws an exception if the exit code of the script is nonzero */ - public Result bash(String script) { - return run(bashCommand(script)); + public Result run(String script) { + return runScript(getScriptCommand(script)); } /** - * Runs a script in a bash shell + * Same as {@link #run(String)}, but does not throw an exception if the exit code of the script is nonzero */ - public Result bashIgnoreExitCode(String script) { - return runIgnoreExitCode(bashCommand(script)); + public Result runIgnoreExitCode(String script) { + return runScriptIgnoreExitCode(getScriptCommand(script)); } - private static String[] bashCommand(String script) { - return Stream.concat(Stream.of("bash", "-c"), Stream.of(script)).toArray(String[]::new); - } - - /** - * Runs a script in a powershell shell, throwing an exception if its exit code is nonzero - */ - public Result powershell(String script) { - return run(powershellCommand(script)); + private String[] getScriptCommand(String script) { + if (Platforms.WINDOWS) { + return powershellCommand(script); + } else { + return bashCommand(script); + } } - /** - * Runs a script in a powershell shell - */ - public Result powershellIgnoreExitCode(String script) { - return runIgnoreExitCode(powershellCommand(script)); + private static String[] bashCommand(String script) { + return Stream.concat(Stream.of("bash", "-c"), Stream.of(script)).toArray(String[]::new); } private static String[] powershellCommand(String script) { return Stream.concat(Stream.of("powershell.exe", "-Command"), Stream.of(script)).toArray(String[]::new); } - /** - * Runs an executable file, passing all elements of {@code command} after the first as arguments. Throws an exception if the process' - * exit code is nonzero - */ - private Result run(String[] command) { - Result result = runIgnoreExitCode(command); + private Result runScript(String[] command) { + Result result = runScriptIgnoreExitCode(command); if (result.isSuccess() == false) { throw new RuntimeException("Command was not successful: [" + String.join(" ", command) + "] result: " + result.toString()); } return result; } - /** - * Runs an executable file, passing all elements of {@code command} after the first as arguments - */ - private Result runIgnoreExitCode(String[] command) { + private Result runScriptIgnoreExitCode(String[] command) { ProcessBuilder builder = new ProcessBuilder(); builder.command(command); From 8012b7871b407743ba8698e7835190ef6ef173df Mon Sep 17 00:00:00 2001 From: Andy Bristol Date: Thu, 5 Jul 2018 11:06:47 -0700 Subject: [PATCH 5/5] code review fixes --- .../packaging/test/ArchiveTestCase.java | 19 +++++++++++-------- .../packaging/util/Platforms.java | 12 ++++++++++-- .../packaging/util/ServerUtils.java | 5 +++-- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java index 684df26ab979e..df5e8cf995d86 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/test/ArchiveTestCase.java @@ -134,12 +134,15 @@ public void test30AbortWhenJavaMissing() { Platforms.onLinux(() -> { final String javaPath = sh.run("which java").stdout.trim(); - sh.run("chmod -x '" + javaPath + "'"); - final Result runResult = sh.runIgnoreExitCode(bin.elasticsearch.toString()); - sh.run("chmod +x '" + javaPath + "'"); - assertThat(runResult.exitCode, is(1)); - assertThat(runResult.stdout, containsString("could not find java; set JAVA_HOME or ensure java is in PATH")); + try { + sh.run("chmod -x '" + javaPath + "'"); + final Result runResult = sh.runIgnoreExitCode(bin.elasticsearch.toString()); + assertThat(runResult.exitCode, is(1)); + assertThat(runResult.stdout, containsString("could not find java; set JAVA_HOME or ensure java is in PATH")); + } finally { + sh.run("chmod +x '" + javaPath + "'"); + } }); } @@ -175,15 +178,15 @@ public void test40CreateKeystoreManually() { final Result r = sh.run(bin.elasticsearchKeystore + " list"); assertThat(r.stdout, containsString("keystore.seed")); }); - - // cleanup for next test - rm(installation.config("elasticsearch.keystore")); } @Test public void test50StartAndStop() throws IOException { assumeThat(installation, is(notNullValue())); + // cleanup from previous test + rm(installation.config("elasticsearch.keystore")); + Archives.runElasticsearch(installation); final String gcLogName = Platforms.LINUX diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Platforms.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Platforms.java index a388231c30382..5ffbc31820022 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Platforms.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/Platforms.java @@ -66,15 +66,23 @@ public static boolean isSysVInit() { return new Shell().runIgnoreExitCode("which service").isSuccess(); } - public static void onWindows(Runnable action) { + public static void onWindows(PlatformAction action) { if (WINDOWS) { action.run(); } } - public static void onLinux(Runnable action) { + public static void onLinux(PlatformAction action) { if (LINUX) { action.run(); } } + + /** + * Essentially a Runnable, but we make the distinction so it's more clear that these are synchronous + */ + @FunctionalInterface + public interface PlatformAction { + void run(); + } } diff --git a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/ServerUtils.java b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/ServerUtils.java index 0fc9ddb57849d..ff006a34e6892 100644 --- a/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/ServerUtils.java +++ b/qa/vagrant/src/main/java/org/elasticsearch/packaging/util/ServerUtils.java @@ -39,6 +39,7 @@ public class ServerUtils { private static final Log LOG = LogFactory.getLog(ServerUtils.class); private static final long waitTime = TimeUnit.SECONDS.toMillis(60); + private static final long timeoutLength = TimeUnit.SECONDS.toMillis(10); public static void waitForElasticsearch() throws IOException { waitForElasticsearch("green", null); @@ -56,8 +57,8 @@ public static void waitForElasticsearch(String status, String index) throws IOEx try { final HttpResponse response = Request.Get("http://localhost:9200/_cluster/health") - .connectTimeout((int) waitTime) - .socketTimeout((int) waitTime) + .connectTimeout((int) timeoutLength) + .socketTimeout((int) timeoutLength) .execute() .returnResponse();