diff --git a/agent/org.linkedin.glu.agent-cli-impl/src/main/groovy/org/linkedin/glu/agent/cli/ClientMain.groovy b/agent/org.linkedin.glu.agent-cli-impl/src/main/groovy/org/linkedin/glu/agent/cli/ClientMain.groovy index 541fe2e8..c57e32af 100644 --- a/agent/org.linkedin.glu.agent-cli-impl/src/main/groovy/org/linkedin/glu/agent/cli/ClientMain.groovy +++ b/agent/org.linkedin.glu.agent-cli-impl/src/main/groovy/org/linkedin/glu/agent/cli/ClientMain.groovy @@ -21,6 +21,7 @@ package org.linkedin.glu.agent.cli import org.linkedin.glu.agent.api.Agent import org.linkedin.glu.agent.rest.client.AgentFactory import org.linkedin.glu.agent.rest.client.AgentFactoryImpl +import org.linkedin.glu.groovy.utils.io.GluGroovyIOUtils import org.linkedin.groovy.util.config.Config import org.linkedin.groovy.util.state.StateMachine import org.linkedin.util.lifecycle.Startable @@ -168,7 +169,7 @@ class ClientMain implements Startable InputStream mis = agent.streamCommandResults(args).stream - exitValue = AgentRestUtils.demultiplexExecStream(mis, System.out, System.err) as int + exitValue = GluGroovyIOUtils.demultiplexExecStream(mis, System.out, System.err) as int } boolean waitForCommandNoTimeOutException(Agent agent, def args) diff --git a/agent/org.linkedin.glu.agent-impl/build.gradle b/agent/org.linkedin.glu.agent-impl/build.gradle index e32ca353..9836e4a7 100644 --- a/agent/org.linkedin.glu.agent-impl/build.gradle +++ b/agent/org.linkedin.glu.agent-impl/build.gradle @@ -36,7 +36,6 @@ dependencies { runtime spec.external.ivy testRuntime project(':utils:org.linkedin.glu.utils.log4j-test-config') - testRuntime spec.external.ivy } idea.module.iml { diff --git a/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/capabilities/ShellExec.groovy b/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/capabilities/ShellExec.groovy index e82f7758..e880361b 100644 --- a/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/capabilities/ShellExec.groovy +++ b/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/capabilities/ShellExec.groovy @@ -18,7 +18,7 @@ package org.linkedin.glu.agent.impl.capabilities import org.apache.tools.ant.taskdefs.Execute import org.linkedin.glu.agent.api.ShellExecException -import org.linkedin.glu.commands.impl.StreamType +import org.linkedin.glu.groovy.utils.io.StreamType import org.linkedin.glu.groovy.utils.GluGroovyLangUtils import org.linkedin.glu.groovy.utils.io.InputGeneratorStream import org.linkedin.glu.groovy.utils.json.GluGroovyJsonUtils diff --git a/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/capabilities/ShellImpl.groovy b/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/capabilities/ShellImpl.groovy index ee752078..2101ad64 100644 --- a/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/capabilities/ShellImpl.groovy +++ b/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/capabilities/ShellImpl.groovy @@ -18,86 +18,25 @@ package org.linkedin.glu.agent.impl.capabilities -import eu.medsea.mimeutil.MimeUtil -import org.linkedin.glu.groovy.utils.collections.GluGroovyCollectionUtils -import org.linkedin.groovy.util.ant.AntUtils - -import java.util.concurrent.TimeoutException -import java.util.regex.Pattern -import javax.management.MBeanServerConnection -import javax.management.ObjectName -import javax.management.remote.JMXConnectorFactory -import javax.management.remote.JMXServiceURL import org.linkedin.glu.agent.api.ScriptFailedException import org.linkedin.glu.agent.api.Shell -import org.linkedin.groovy.util.concurrent.GroovyConcurrentUtils -import org.linkedin.groovy.util.io.fs.FileSystem -import org.linkedin.util.clock.Clock -import org.linkedin.util.clock.SystemClock -import org.linkedin.groovy.util.net.GroovyNetUtils -import org.linkedin.groovy.util.encryption.EncryptionUtils -import org.linkedin.groovy.util.io.GroovyIOUtils -import org.linkedin.util.lang.MemorySize -import org.apache.tools.ant.filters.ReplaceTokens -import org.apache.tools.ant.filters.ReplaceTokens.Token -import org.linkedin.util.io.resource.Resource -import javax.management.Attribute import org.linkedin.glu.agent.impl.storage.AgentProperties -import org.linkedin.util.url.QueryBuilder -import org.linkedin.groovy.util.rest.RestException -import org.linkedin.util.reflect.ReflectUtils -import org.linkedin.glu.agent.rest.common.AgentRestUtils -import org.linkedin.glu.groovy.utils.concurrent.FutureTaskExecutionThreadFactory -import org.linkedin.glu.utils.concurrent.Submitter -import org.linkedin.glu.utils.concurrent.OneThreadPerTaskSubmitter /** * contains the utility methods for the shell * * @author ypujante@linkedin.com */ -def class ShellImpl implements Shell +def class ShellImpl extends org.linkedin.glu.groovy.utils.shell.ShellImpl implements Shell { public static final String MODULE = ShellImpl.class.getName(); public static final org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(MODULE); - private static final String CONNECTOR_ADDRESS = - "com.sun.management.jmxremote.localConnectorAddress"; - - static { - MimeUtil.registerMimeDetector("eu.medsea.mimeutil.detector.MagicMimeMimeDetector"); - } - - Clock clock = SystemClock.instance() - - // will delegate all the calls to fileSystem - @Delegate FileSystem fileSystem - // agent properties AgentProperties agentProperties - /** - * The charset to use for reading/writing files */ - def charset = 'UTF-8' - - /** - * Used whe threads are needed - */ - private Submitter _submitter - - synchronized Submitter getSubmitter() - { - if(_submitter == null) - _submitter = new OneThreadPerTaskSubmitter(new FutureTaskExecutionThreadFactory()) - return _submitter - } - - void setSubmitter(Submitter submitter) - { - _submitter = submitter - } - - Shell newShell(fileSystem) + @Override + ShellImpl newShell(fileSystem) { return new ShellImpl(fileSystem: fileSystem, agentProperties: agentProperties, @@ -111,898 +50,25 @@ def class ShellImpl implements Shell return agentProperties?.exposedProperties ?: [:] } - Collection getMimeTypes(file) - { - withInputStream(file) { InputStream is -> - MimeUtil.getMimeTypes(is).collect { it.toString() } - } as Collection - } - - Resource unzip(file) - { - return unzip(file, createTempDir()) - } - - Resource unzip(file, toDir) - { - file = toResource(file) - toDir = toResource(toDir) - ant { ant -> - ant.unzip(src: file.file, dest: toDir.file) - } - return toDir - } - - Resource untar(file) - { - return untar(file, createTempDir()) - } - - Resource untar(file, toDir) - { - file = toResource(file) - toDir = toResource(toDir) - def compression = 'none' - - def mimeTypes = getMimeTypes(file) - - if(mimeTypes.find { it == 'application/x-gzip'}) - { - compression = 'gzip' - } - - ant { ant -> - ant.untar(src: file.file, dest: toDir.file, compression: compression) - } - - return toDir - } - - /** - * Uncompresses the provided file - * @return the original file or the new one - */ - Resource gunzip(file) - { - file = toResource(file) - - def gunzipFilename - if(file.filename.endsWith('.gz')) - { - gunzipFilename = file.filename[0..-4] - } - else - { - if(file.filename.endsWith('.tgz')) - { - gunzipFilename = "${file.filename[0..-5]}.tar" - } - else - { - throw new IOException("unknown suffix ${file.filename}") - } - } - - def gunzipFile = gunzip(file, file.parentResource."${URLEncoder.encode(gunzipFilename)}") - - if(file != gunzipFile) - { - rm(file) - } - - return gunzipFile - } - - /** - * Uncompresses the provided file (if compressed) into a new file - */ - Resource gunzip(file, toFile) - { - file = toResource(file) - toFile = toResource(toFile) - - if(!file.isDirectory() && getMimeTypes(file).find { it == 'application/x-gzip'}) - { - if(!toFile.exists()) - { - mkdirs(toFile.parentResource) - } - - ant { ant -> - ant.gunzip(src: file.file, dest: toFile.file) - } - - return toFile - } - else - { - return file - } - } - - /** - * Compresses the provided file (first make sure that the file is compressed) - * - * @return the gziped file or the original file if not zipped - */ - Resource gzip(file) - { - file = toResource(file) - - def gzipFile = gzip(file, file.parentResource."${URLEncoder.encode(file.filename)}.gz") - - if(file != gzipFile) - { - rm(file) - } - - return gzipFile - } - - /** - * Compresses the provided file (first make sure that the file is compressed) - * - * @return the gziped file or the original file if not zipped - */ - Resource gzip(file, toFile) - { - file = toResource(file) - toFile = toResource(toFile) - - if(!file.isDirectory() && getMimeTypes(file).find { it != 'application/x-gzip'}) - { - ant { ant -> - ant.gzip(src: file.file, destfile: toFile.file) - } - - return toFile - } - else - { - return file - } - } - - /** - * Compresses each file in a folder - * - * @param fileOrFolder a file (behaves like {@link #gzip(Object)}) or a folder - * @param recurse if true then recursively process all folders - * @return a map with key is resource and value is delta in size (original - new) (note that it - * contains only the resources that are modified!) - */ - Map gzipFileOrFolder(fileOrFolder, boolean recurse) - { - def res = [:] - - fileOrFolder = toResource(fileOrFolder) - - if(fileOrFolder.isDirectory()) - { - fileOrFolder.ls().each { file -> - if(recurse) - { - res.putAll(gzipFileOrFolder(file, recurse)) - } - else - { - long size = file.size() - def gzipedFile = gzip(file) - if(gzipedFile != file) - res[gzipedFile] = size - gzipedFile.size() - } - } - } - else - { - long size = fileOrFolder.size() - def gzipedFile = gzip(fileOrFolder) - if(gzipedFile != fileOrFolder) - res[gzipedFile] = size - gzipedFile.size() - } - - return res - } - - Resource untarAndDecrypt(file, toDir, encryptionKeys) - { - - def tmpDir = createTempDir() - untar(file, tmpDir) - EncryptionUtils.decryptFiles(tmpDir.getFile(), toDir.getFile(), encryptionKeys) - rmdirs(tmpDir) - return toDir - } - - - /** - * Exporting ant access to the shell - */ - def ant(Closure closure) - { - return AntUtils.withBuilder { closure(it) } - } - - /** - * Fetches the file pointed to by the location. The location can be File, - * a String or URI and must contain a scheme. Example of locations: - * http://locahost:8080/file.txt', file:/tmp/file.txt, - * ivy:/org.linkedin/util-core/1.0.0. - */ - Resource fetch(location) - { - return fetch(location, null) - } - - /** - * Fetches the file pointed to by the location. The location can be File, - * a String or URI and must contain a scheme. Example of locations: - * http://locahost:8080/file.txt', file:/tmp/file.txt, - * ivy:/org.linkedin/util-core/1.0.0. The difference with the other fetch method - * is that it fetches the file in the provided destination rather than in the tmp space. - */ - Resource fetch(location, destination) - { - URI uri = GroovyNetUtils.toURI(location) - - if(uri == null) - return null - - Resource tempFile - if(destination) - { - tempFile = toResource(destination) - mkdirs(tempFile.parentResource) - } - else - { - tempFile = createTempDir() - } - - if(tempFile.isDirectory()) - { - def filename = GroovyNetUtils.guessFilename(uri) - tempFile = tempFile.createRelative(filename) - } - - GroovyIOUtils.fetchContent(location, tempFile.file) - - return tempFile - } - - /** - * Fetches the content of the location and returns it as a String or - * null if the location is not reachable - * - * @deprecated use {@link #cat(Object) instead - */ - String fetchContent(location) - { - return cat(location) - } - - /** - * Returns the content of the location as a String or - * null if the location is not reachable - */ - String cat(location) - { - try - { - return GroovyIOUtils.cat(location) - } - catch(Exception e) - { - if(log.isDebugEnabled()) - log.debug("[ignored] exception while catting content ${location}", e) - return null - } - } - - /** - * Issue a 'HEAD' request. The location should be an http or https link. - * - * @param location - * @return a map with the following entries: - * responseCode: 200, 404... {@link java.net.HttpURLConnection#getResponseCode()} - * responseMessage: message {@link java.net.HttpURLConnection#getResponseMessage()} - * headers: representing all the headers {@link java.net.URLConnection#getHeaderFields()} - */ - Map httpHead(location) - { - Map res = [:] - - URI uri = GroovyNetUtils.toURI(location) - - URL url = uri.toURL() - URLConnection cx = url.openConnection() - try - { - if(cx instanceof HttpURLConnection) - { - cx.requestMethod = 'HEAD' - cx.doInput = true - cx.doOutput = false - - cx.connect() - - res.responseCode = cx.responseCode - res.responseMessage = cx.responseMessage - res.headers = cx.headerFields - } - } - finally - { - if(cx.respondsTo('close')) - cx.close() - } - - return res - } - - /** - * Issue a 'POST' request. The location should be an http or https link. The request will be - * made with application/x-www-form-urlencoded content type. - * - * @param location - * @param parameters the parameters of the post as map of key value pairs (value can be a single - * value or a collection of values) - * @return a map with the following entries: - * responseCode: 200, 404... {@link java.net.HttpURLConnection#getResponseCode()} - * responseMessage: message {@link java.net.HttpURLConnection#getResponseMessage()} - * headers: representing all the headers {@link java.net.URLConnection#getHeaderFields()} - */ - Map httpPost(location, Map parameters) - { - Map res = [:] - - URI uri = GroovyNetUtils.toURI(location) - - URL url = uri.toURL() - URLConnection cx = url.openConnection() - try - { - if(cx instanceof HttpURLConnection) - { - cx.requestMethod = 'POST' - cx.setRequestProperty('Content-Type', 'application/x-www-form-urlencoded') - cx.doInput = true - cx.doOutput = true - - cx.connect() - - QueryBuilder qb = new QueryBuilder() - - parameters?.each { k, v -> - if(v != null) - { - if(v instanceof Collection) - qb.addParameters(k.toString(), v.collect { it.toString() } as String[]) - else - qb.addParameter(k.toString(), v.toString()) - } - } - - cx.outputStream << qb.toString() - cx.outputStream.close() - - res.responseCode = cx.responseCode - res.responseMessage = cx.responseMessage - res.headers = cx.headerFields - } - } - finally - { - if(cx.respondsTo('close')) - cx.close() - } - - return res - } - - /** - * Similarly to the unix grep command, checks the location one line at a time and returns - * all the lines which matches the pattern. - */ - Collection grep(location, pattern) - { - return (Collection) grep(location, pattern, null) - } - - /** - * Similarly to the unix grep command, checks the location one line at a time and returns - * all the lines which matches the pattern - * - * @param options options to the command: - * out: an object to output the lines which match (default to []) - * count: returns the count only (does not use out) - * maxCount: stop reading after maxCount matches - */ - def grep(location, pattern, options) - { - options = options ?: [:] - options = new HashMap(options) - def out = options.out ?: [] - int count = 0 - - def uri = GroovyNetUtils.toURI(location) - - if(pattern instanceof String) - { - pattern = Pattern.compile(pattern) - } - - try - { - def idx = 1 - GroovyIOUtils.eachLine(uri.toURL()) { line -> - if(pattern.matcher(line).find()) - { - count++ - if(!options.count) - out << line - if(count == options.maxCount) - return false - } - idx++ - return true - } - } - catch(Exception e) - { - if(log.isDebugEnabled()) - log.debug("[ignored] exception while grepping content ${location}", e) - } - - if(options.count) - { - return count - } - else - { - return out - } - } - - /** - * Executes a shell command... the command will be delegated straight to shell and the output of - * the shell command is returned. - */ - String exec(executable, executableArgs) - { - executable = chmodPlusX(executable) - doExec("${executable} ${toStringCommandLine(executableArgs)}".toString()) - } - - /** - * Executes a shell command... the command will be delegated straight to shell and the output of - * the shell command is returned. - */ - String exec(executable, Object... executableArgs) - { - exec(executable, executableArgs.collect { it }) - } - - /** - * Executes a shell command... the command will be delegated straight to shell and the output of - * the shell command is returned. - */ - String exec(String command) - { - return doExec(command) - } - - /** - * Executes a shell command... the command will be delegated straight to shell and the output of - * the shell command is returned. - */ - String exec(Collection command) - { - return doExec(command) - } - /** * @{inheritDoc} */ def exec(Map args) { - if(args.pwd) - { - def pwd = toResource(args.pwd).file - args = GluGroovyCollectionUtils.xorMap(args, ['pwd']) - args.pwd = pwd - } - new ShellExec(shell: this, args: args).exec() - } - - /** - * @{inheritDoc} - */ - def demultiplexExecStream(InputStream execStream, - OutputStream stdout, - OutputStream stderr) - { - AgentRestUtils.demultiplexExecStream(execStream, stdout, stderr) - } - - /** - * This method will try to rebuild the full stack trace based on the rest exception recursively. - * Handles the case when the client does not know about an exception - * (or it simply cannot be created). - */ - private Throwable doRebuildAgentException(RestException restException) - { - Throwable originalException = restException try { - def exceptionClass = ReflectUtils.forName(restException.originalClassName) - originalException = exceptionClass.newInstance([restException.originalMessage] as Object[]) - - originalException.setStackTrace(restException.stackTrace) - - if(restException.cause) - originalException.initCause(doRebuildAgentException(restException.cause)) + super.exec(args) } - catch(Exception e) + catch(org.linkedin.glu.groovy.utils.shell.ShellExecException e) { - if(log.isDebugEnabled()) - { - log.debug("Cannot instantiate: ${restException.originalClassName}... ignored", e) - } - } - - return originalException - } - - /** - * Shortcut/More efficient implementation of the more generic {@link #chmod(Object, Object) call - */ - Resource chmodPlusX(file) - { - file = toResource(file) - file.file.setExecutable(true) - return file - } - - /** - * Change the permission of the file - */ - Resource chmod(file, perm) - { - file = toResource(file) - exec(command: ['chmod', perm, file.file], res: 'exitValue') - return file - } - - Resource chmodRecursive(dir, perm) - { - eachChildRecurse(dir) { file -> - exec(command: ['chmod', perm, file.file], res: 'exitValue') - } - - return dir - } - - /** - * Invokes the closure with an MBeanServerConnection to the jmx control running - * on the vm started with the provided pid. The closure will be invoked with null - * if cannot determine the process. - */ - def withMBeanServerConnection(pid, Closure closure) - { - def serviceURL = extractJMXServiceURL(pid) - if(serviceURL) - { - def connector = null - try - { - connector = JMXConnectorFactory.connect(serviceURL) - } - catch(IOException e) - { - // ignored (connector remains null) - if(log.isDebugEnabled()) - { - log.debug("Ignored exception", e) - } - } - - try - { - use(MBeanServerConnectionCategory) { - closure(connector?.MBeanServerConnection) - } - } - finally - { - connector?.close() - } - } - else - { - closure(null) - } - } - - /** - * Extract the serviceURL using sun internal implementation - */ - private JMXServiceURL extractJMXServiceURL(pid) - { - if(pid == null) - return null - - String serviceURL = null - try - { - serviceURL = sun.management.ConnectorAddressLink.importFrom(pid as int) - } - catch (IOException e) - { - log.warn("Cannot find process ${pid}") - } - - if(serviceURL == null) - return null - else - return new JMXServiceURL(serviceURL) - } - - /** - * Waits for the condition to be true no longer than the timeout. true - * is returned if the condition was satisfied, false otherwise (if you specify - * noException) - * @param args.timeout how long max to wait - * @param args.heartbeat how long to wait between calling the condition - * @param args.noException to get false instead of an exception - */ - boolean waitFor(args, Closure condition) - { - try - { - GroovyConcurrentUtils.waitForCondition(clock, args.timeout, args.heartbeat, condition) - return true - } - catch (TimeoutException e) - { - if(args.noException) - return false - else - throw e - } - } - - /** - * Waits for the condition to be true no longer than the timeout. true - * is returned if the condition was satisfied, false otherwise - */ - boolean waitFor(Closure condition) - { - return waitFor([:], condition) - } - - /** - * Tail the location - * - * @params location the location of the file to tail - * @params maxLine the number of lines maximum to read - * - * @return the input stream to read the content of the tail - */ - InputStream tail(location, long maxLine) - { - return tail([location: location, maxLine: maxLine]) - } - - /** - * Tail the location - * - * @params args.location the location of the file to tail - * @params args.maxLine the number of lines maximum to read (-1 for the entire file) - * @params args.maxSize the maximum size to read (-1 for the entire file) - * - * @return the input stream to read the content of the tail - */ - InputStream tail(args) - { - File file = toResource(args.location)?.file - - if(file && file.exists()) - { - if(args.maxLine?.toString() == '-1' || args.maxSize?.toString() == '-1') - return new FileInputStream(file) - - def commandLine = ['tail'] - if(args.maxLine) - commandLine << "-${args.maxLine}" - if(args.maxSize) - commandLine << '-c' << MemorySize.parse(args.maxSize.toString()).sizeInBytes - commandLine << file.canonicalPath - return forkAndExec(commandLine) - } - else - return null - } - - /** - * Forks a process to execute the command line provided (as a single string) and returns the - * input stream of the process - */ - private InputStream forkAndExec(commandLine) - { - exec(command: commandLine, - redirectStderr: true, - res: 'stdoutStream') as InputStream - } - - /** - * Make sure that the command line is a string. - */ - protected String toStringCommandLine(commandLine) - { - if(commandLine instanceof GString) - commandLine = commandLine.toString() - - if(!(commandLine instanceof String)) - commandLine = commandLine.collect { it.toString() }.join(' ') - - return commandLine - } - - protected InputStream toInputStream(stream) - { - if(stream == null) - return stream - - if(stream instanceof InputStream) - return stream - - return new ByteArrayInputStream(stream.toString().getBytes(charset)) - } - - private def doExec(commandLine) - { - return exec(command: commandLine, - failOnError: true, - res: 'all').stdout - } - - /** - * Converts the output into a string. Assumes that the encoding is UTF-8. Replaces all line feeds - * by '\n' and remove the last line feed. - */ - protected def toStringOutput(byte[] output) - { - return output ? new String(output, charset).readLines().join('\n') : "" - } - - /** - * Converts the output into a string with no more than maxChars characters. If there are more - * characters, then display '...' at the end. - */ - private def toLimitedStringOutput(byte[] output, int maxChars) - { - def string = toStringOutput(output) - if(string?.size() > maxChars) - { - string = string.substring(0, maxChars) + "..." - } - return string - } - - /** - * @return true if there is a socket open on the server/port combination - */ - boolean listening(server, port) - { - ant { ant -> - ant.condition(property: 'listening') { - socket(server: server, port: port) - }.project.getProperty('listening') ? true : false - } as boolean - } - - /** - * Replaces the tokens provided in the map in the input. Token replacement is using ant token - * replacement class so in the input, the tokens are surrounded by the '@' sign. - * Example: - * - * 
-   * input = "abcd @myToken@"
-   * assert "abcd foo" == replaceTokens(input, [myToken: 'foo'])
-   *
- */ - String replaceTokens(String input, Map tokens) - { - if(input == null) - return null - - ReplaceTokens rt = new ReplaceTokens(new StringReader(input)) - tokens?.each { k,v -> - rt.addConfiguredToken(new Token(key: k, value: v)) - } - - return rt.text - } - - /** - * Processes from through the replacement token mechanism and writes the result to - * to - * - * @param from anything that can be provided to {@link FileSystem#toResource(Object)} - * @param to anything that can be provided to {@link FileSystem#toResource(Object)} - * @param tokens a map of token - * @return to as a {@link Resource} - * @see #replaceTokens(String, Map) - */ - Resource replaceTokens(def from, def to, Map tokens) - { - to = toResource(to) - - withWriter(to) { Writer writer -> - withReader(from) { Reader reader -> - ReplaceTokens rt = new ReplaceTokens(reader) - tokens?.each { k,v -> - rt.addConfiguredToken(new Token(key: k, value: v)) - } - writer << rt - } - } - - return to - } - - /** - * Processes the content to the token replacement method. - * - * @see FileSystem#saveContent(Object, String) - */ - Resource saveContent(file, String content, Map tokens) - { - saveContent(file, replaceTokens(content, tokens)) - } - - /** - * Same as withInputStream but wraps in a reader using the {@link #charset} - * @param file anything that can be provided to {@link FileSystem#toResource(Object)} - * @return whatever the closure returns - */ - def withReader(file, Closure closure) - { - withInputStream(file) { InputStream is -> - is.withReader(charset, closure) - } - } - - /** - * Same as withOutputStream but wraps in a writer using the {@link #charset} - * @param file anything that can be provided to {@link FileSystem#toResource(Object)} - * @return whatever the closure returns - */ - def withWriter(file, Closure closure) - { - withOutputStream(file) { OutputStream os -> - os.withWriter(charset, closure) - } - } - - /** - * Runs the closure in a protected block that will not throw an exception but will return - * null in the case one happens - */ - def noException(Closure closure) - { - try - { - return closure() - } - catch(Throwable th) - { - if(log.isDebugEnabled()) - { - log.debug("[ignored] exception", e) - } - return null + // need to rethrow the exception to match the api! + def ne = new org.linkedin.glu.agent.api.ShellExecException(e.message) + ne.res = e.res + ne.output = e.output + ne.error = e.error + ne.initCause(e.cause) + e.suppressed.each { ne.addSuppressed(it) } + throw ne } } @@ -1014,59 +80,3 @@ def class ShellImpl implements Shell throw new ScriptFailedException(message?.toString()) } } - -class MBeanServerConnectionCategory -{ - static def getAttribute(MBeanServerConnection self, objectName, String attribute) - { - objectName = toObjectName(objectName) - - return self.getAttribute(objectName, attribute) - } - - static Map getAttributes(MBeanServerConnection self, objectName, attributes) - { - objectName = toObjectName(objectName) - - attributes = attributes as String[] - - Map res = [:] - - self.getAttributes(objectName, attributes)?.each { Attribute attribute -> - res[attribute.name] = attribute.value - } - - return res - } - - static def invoke(MBeanServerConnection self, objectName, String methodName, parameters) - { - objectName = toObjectName(objectName) - - def info = self.getMBeanInfo(objectName) - - def operations = info.getOperations().findAll { - it.name == methodName && it.signature.size() == parameters.size() - } - - // if there is only one method with the given name the same number of parameters then we can call - // it - if(operations.size() != 1) - { - if(operations) - throw new UnsupportedOperationException("cannot use this form of invoke for overloaded methods") - else - throw new NoSuchMethodException(methodName) - } - - self.invoke(objectName, methodName, parameters as Object[], operations[0].signature.type as String[]) - } - - private static ObjectName toObjectName(objectName) - { - if(!(objectName instanceof ObjectName)) - objectName = new ObjectName(objectName.toString()) - - return objectName - } -} diff --git a/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/command/CommandGluScript.groovy b/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/command/CommandGluScript.groovy index cd935b94..5d3fb15b 100644 --- a/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/command/CommandGluScript.groovy +++ b/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/command/CommandGluScript.groovy @@ -18,7 +18,7 @@ package org.linkedin.glu.agent.impl.command import org.linkedin.glu.commands.impl.CommandExecution import org.linkedin.glu.commands.impl.CommandStreamStorage -import org.linkedin.glu.commands.impl.StreamType +import org.linkedin.glu.groovy.utils.io.StreamType import org.linkedin.glu.groovy.utils.concurrent.CallExecution import org.linkedin.glu.commands.impl.CommandExecutionIOStorage import org.linkedin.util.annotations.Initializable diff --git a/agent/org.linkedin.glu.agent-impl/src/test/groovy/test/agent/impl/TestAgentImpl.groovy b/agent/org.linkedin.glu.agent-impl/src/test/groovy/test/agent/impl/TestAgentImpl.groovy index dfcac2a3..cb0ed321 100644 --- a/agent/org.linkedin.glu.agent-impl/src/test/groovy/test/agent/impl/TestAgentImpl.groovy +++ b/agent/org.linkedin.glu.agent-impl/src/test/groovy/test/agent/impl/TestAgentImpl.groovy @@ -24,7 +24,6 @@ import org.linkedin.glu.agent.impl.AgentImpl import org.linkedin.glu.agent.impl.capabilities.ShellImpl import org.linkedin.glu.agent.impl.script.FromClassNameScriptFactory import org.linkedin.glu.agent.impl.script.RootScript -import org.linkedin.glu.agent.impl.script.ScriptDefinition import org.linkedin.glu.agent.impl.storage.RAMStorage import org.linkedin.glu.agent.impl.storage.Storage import org.linkedin.glu.agent.api.ScriptExecutionException diff --git a/agent/org.linkedin.glu.agent-impl/src/test/groovy/test/agent/impl/TestCapabilities.groovy b/agent/org.linkedin.glu.agent-impl/src/test/groovy/test/agent/impl/TestCapabilities.groovy index 9791a476..45a13154 100644 --- a/agent/org.linkedin.glu.agent-impl/src/test/groovy/test/agent/impl/TestCapabilities.groovy +++ b/agent/org.linkedin.glu.agent-impl/src/test/groovy/test/agent/impl/TestCapabilities.groovy @@ -1,6 +1,6 @@ /* * Copyright (c) 2010-2010 LinkedIn, Inc - * Portions Copyright (c) 2011-2012 Yan Pujante + * Portions Copyright (c) 2011-2013 Yan Pujante * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -18,128 +18,23 @@ package test.agent.impl -import org.linkedin.glu.agent.impl.capabilities.ShellExec -import org.linkedin.glu.agent.impl.capabilities.ShellImpl -import org.linkedin.groovy.util.io.fs.FileSystemImpl -import org.linkedin.groovy.util.io.fs.FileSystem -import org.linkedin.util.io.resource.Resource -import org.linkedin.groovy.util.net.SingletonURLStreamHandlerFactory -import org.linkedin.groovy.util.ivy.IvyURLHandler -import org.linkedin.groovy.util.io.GroovyIOUtils -import org.linkedin.groovy.util.collections.GroovyCollectionsUtils -import org.linkedin.groovy.util.net.GroovyNetUtils import org.linkedin.glu.agent.api.ShellExecException +import org.linkedin.glu.agent.impl.capabilities.ShellImpl import org.linkedin.glu.agent.impl.storage.AgentProperties -import com.sun.net.httpserver.HttpExchange -import com.sun.net.httpserver.Headers +import org.linkedin.groovy.util.io.fs.FileSystem +import org.linkedin.groovy.util.io.fs.FileSystemImpl /** - * Test for various capabilities. + * Test for various capabilities. Most of the test has been moved under the utils module + * for the generic shell * - * @author ypujante@linkedin.com + * @author yan@pongasoft.com */ class TestCapabilities extends GroovyTestCase { public static final String MODULE = TestCapabilities.class.getName(); public static final org.slf4j.Logger log = org.slf4j.LoggerFactory.getLogger(MODULE); - void testFetch() - { - FileSystemImpl.createTempFileSystem() { FileSystem fs -> - def shell = new ShellImpl(fileSystem: fs) - - Resource tempFile = fs.tempFile() - assertFalse(tempFile.exists()) - fs.saveContent(tempFile, "this is a test") - assertTrue(tempFile.exists()) - - // on the other end if we provide a uri it will 'fetch' it to a different location - Resource fetchedFile = shell.fetch(tempFile) - assertNotSame(tempFile.file.canonicalPath, fetchedFile.file.canonicalPath) - - // we make sure that the file was copied entirely - assertEquals("this is a test", fs.readContent(fetchedFile)) - - // we now remove the temp file and we make sure that fetch throws an exception - fs.rm(tempFile) - assertFalse(tempFile.exists()) - shouldFail(FileNotFoundException) { shell.fetch(tempFile.toURI()) } - } - } - - /** - * Using shell.fetch on a remote url (http) - */ - void testFetchRemote() - { - FileSystemImpl.createTempFileSystem() { FileSystem fs -> - def shell = new ShellImpl(fileSystem: fs) - - String response - Headers requestHeaders - - def handler = { HttpExchange t -> - requestHeaders = t.requestHeaders - t.sendResponseHeaders(200, response.length()); - OutputStream os = t.getResponseBody(); - os.write(response.getBytes()); - os.close(); - } - - GroovyNetUtils.withHttpServer(0, ['/content': handler]) { int port -> - - File root = fs.root.file - - File tmpFile = new File(root, 'foo.txt') - - response = "abc" - - shell.fetch("http://localhost:${port}/content", tmpFile) - assertEquals("abc", tmpFile.text) - // no authorization header should be present! - assertFalse(requestHeaders.containsKey('Authorization')) - - response = "def" - shell.fetch("http://u1:p1@localhost:${port}/content", tmpFile) - assertEquals("def", tmpFile.text) - // authorization header should be present and properly base64ified - assertEquals("Basic ${'u1:p1'.bytes.encodeBase64()}", - requestHeaders['Authorization'].iterator().next()) - } - } - } - - /** - * Use a local repo to fetch a file which content is well known. - */ - void testFetchWithIvy() - { - FileSystemImpl.createTempFileSystem() { FileSystem fs -> - - def ivySettings = new File("./src/test/resources/ivysettings.xml").canonicalFile.toURI() - - def factory = new SingletonURLStreamHandlerFactory() - factory.registerHandler('ivy') { - return new IvyURLHandler(ivySettings) - } - URL.setURLStreamHandlerFactory(factory) - - def shell = new ShellImpl(fileSystem: fs) - - def file = shell.fetch('ivy:/test.agent.impl/myArtifact/1.0.0') - - assertEquals(new File("./src/test/resources/test/agent/impl/myArtifact/1.0.0/myArtifact-1.0.0.jar").getText(), - fs.readContent(file)) - - assertEquals('myArtifact-1.0.0.jar', GroovyNetUtils.guessFilename(new URI('ivy:/test.agent.impl/myArtifact/1.0.0'))) - - file = shell.fetch('ivy:/test.agent.impl/myArtifact/1.0.0/text') - - assertEquals(new File("./src/test/resources/test/agent/impl/myArtifact/1.0.0/myArtifact-1.0.0.txt").getText(), - fs.readContent(file)) - } - } - /** * Test that we can exec commands. */ @@ -164,224 +59,6 @@ class TestCapabilities extends GroovyTestCase + shellScriptError, e.message) } - - def shellScript = shell.fetch("./src/test/resources/shellScriptTestCapabilities.sh") - - // we make sure that the script is not executable because exec changes the exec flag - // automatically - fs.chmod(shellScript, '-x') - - // we make sure that the other syntax works - assertEquals("Hello", shell.exec(shellScript)) - assertEquals("Hello a b c", shell.exec(shellScript, "a b c")) - assertEquals("Hello a b c", shell.exec(shellScript, ["a", "b", "c"])) - assertEquals("Hello a b c", shell.exec(shellScript, "a", "b", "c")) - assertEquals("Hello a b c", shell.exec(shellScript.file, ["a", "b", "c"])) - - // and we can call with resource as well - assertEquals("Hello a b c", shell.exec("${shellScript} a b c")) - - // now we try with a file directly: - shellScript = shellScript.file.canonicalPath - assertEquals("Hello a b c", shell.exec("${shellScript} a b c")) - assertEquals(" 1 1 6".trim(), shell.exec("${shellScript} | wc").trim()) - - // we make the shell script non executable - fs.chmod(shellScript, '-x') - try - { - shell.exec("${shellScript} a b c") - fail("should fail") - } - catch(ShellExecException e) - { - assertEquals(126, e.res) - assertEquals('', e.stringOutput) - String shellScriptError = - "bash: ${shellScript}: Permission denied".toString() - assertEquals(shellScriptError, e.stringError) - assertEquals("Error while executing command ${shellScript} a b c: " + - "res=126 - output= - error=${shellScriptError}", - e.message) - } - } - } - - void testGenericExec() - { - FileSystemImpl.createTempFileSystem() { FileSystem fs -> - def shell = new ShellImpl(fileSystem: fs) - def shellScript = shell.fetch("./src/test/resources/shellScriptTestShellExec.sh") - // let's make sure it is executable - fs.chmod(shellScript, '+x') - - // stdout only - checkShellExec(shell, [command: [shellScript, "-1"]], 0, "this goes to stdout\n", "") - - // both stdout and stderr in their proper channel - checkShellExec(shell, [command: [shellScript, "-1", "-2"]], 0, "this goes to stdout\n", "this goes to stderr\n") - - // redirecting stderr to stdout - checkShellExec(shell, [command: [shellScript, "-1", "-2"], redirectStderr: true], 0, "this goes to stdout\nthis goes to stderr\n", "") - - // changing stdout - def myStdout = new ByteArrayOutputStream() - checkShellExec(shell, [command: [shellScript, "-1", "-2"], stdout: myStdout], 0, "", "this goes to stderr\n") { - // implementation note: output here is not "processed" (see javadoc) so need to add the - // final \n character - assertEquals("this goes to stdout\n", new String(myStdout.toByteArray(), "UTF-8")) - myStdout.reset() - } - - // changing stderr - def myStderr = new ByteArrayOutputStream() - checkShellExec(shell, [command: [shellScript, "-1", "-2"], stderr: myStderr], 0, "this goes to stdout\n", "") { - // implementation note: output here is not "processed" (see javadoc) so need to add the - // final \n character - assertEquals("this goes to stderr\n", new String(myStderr.toByteArray(), "UTF-8")) - myStderr.reset() - } - - // testing for failure/exit value - checkShellExec(shell, [command: [shellScript, "-1", "-e"], failOnError: false], 1, "this goes to stdout\n", "") - - // test that when there is a failure, then an exception is properly generated if failOnError - // is not defined - def errorMsg = shouldFail(ShellExecException) { - shell.exec(command: [shellScript, "-1", "-e"]) - } - assertTrue(errorMsg.endsWith("res=1 - output=this goes to stdout - error=")) - - // test that when there is a failure, then an exception is properly generated if failOnError - // is set to true - errorMsg = shouldFail(ShellExecException) { - shell.exec(command: [shellScript, "-1", "-e"], failOnError: true) - } - assertTrue(errorMsg.endsWith("res=1 - output=this goes to stdout - error=")) - - // reading from stdin - checkShellExec(shell, [command: [shellScript, "-1", "-c"], stdin: "abc\ndef\n"], 0, "this goes to stdout\nabc\ndef\n", "") - - // testing for stdoutStream - InputStream stdout = shell.exec(command: [shellScript, "-1", "-2"], res: "stdoutStream") - assertEquals("this goes to stdout\n", stdout.text) - - // testing for stdoutStream - InputStream stderr = shell.exec(command: [shellScript, "-1", "-2"], res: "stderrStream") - assertEquals("this goes to stderr\n", stderr.text) - - // testing for stream - InputStream stream = shell.exec(command: [shellScript, "-1", "-2", "-e"], failOnError: false, res: "stream") - - myStdout = new ByteArrayOutputStream() - myStderr = new ByteArrayOutputStream() - - assertEquals(1, shell.demultiplexExecStream(stream, myStdout, myStderr)) - assertEquals("this goes to stdout\n", new String(myStdout.toByteArray(), "UTF-8")) - assertEquals("this goes to stderr\n", new String(myStderr.toByteArray(), "UTF-8")) - - stream = shell.exec(command: [shellScript, "-1", "-2", "-e"], failOnError: true, res: "stream") - - myStdout = new ByteArrayOutputStream() - myStderr = new ByteArrayOutputStream() - - errorMsg = shouldFail(ShellExecException) { - shell.demultiplexExecStream(stream, myStdout, myStderr) - } - assertTrue(errorMsg.endsWith("res=1 - output= - error=")) - assertEquals("this goes to stdout\n", new String(myStdout.toByteArray(), "UTF-8")) - assertEquals("this goes to stderr\n", new String(myStderr.toByteArray(), "UTF-8")) - - // testing pwd - checkShellExec(shell, [command: ["pwd"]], 0, "${new File(".").canonicalPath}\n", "") - def pwdDir = shell.mkdirs("/pwd") - checkShellExec(shell, [command: ["pwd"], pwd: "/pwd"], 0, "${pwdDir.file.canonicalPath}\n", "") - - ProcessBuilder pb = new ProcessBuilder(ShellExec.buildCommandLine('pwd')) - pb.directory(shell.toResource("/pwdDoNotExist").file) - String errorMessage = null - try - { - pb.start() - } - catch(IOException e) - { - errorMessage = e.message - } - - checkShellExec(shell, [command: ["pwd"], pwd: "/pwdDoNotExist", failOnError: false], 2, "", "${errorMessage}") - - // testing env - checkShellExec(shell, [command: ['echo $HOME']], 0, "${System.getenv().HOME}\n", "") - - // changing environment variable - def homeDir = shell.mkdirs("/home") - checkShellExec(shell, [command: ['echo $HOME'], env: [HOME: homeDir.file.canonicalPath]], 0, "${homeDir.file.canonicalPath}\n", "") - - // removing environment variable - checkShellExec(shell, [command: ['echo $HOME'], env: [HOME: null]], 0, "\n", "") - - // not inheriting - checkShellExec(shell, [command: ['echo $HOME'], inheritEnv: false], 0, "\n", "") - } - } - - private static void checkShellExec(shell, commands, exitValue, stdout, stderr) - { - checkShellExec(shell, commands, exitValue, stdout, stderr, null) - } - - private static void checkShellExec(shell, commands, exitValue, stdout, stderr, Closure cl) - { - assertEquals(stdout.trim(), shell.exec(*:commands)) - if(cl) cl() - assertEquals(exitValue, shell.exec(*:commands, res: "exitValue")) - if(cl) cl() - assertEquals(exitValue.toString(), shell.exec(*:commands, res: "exitValueStream").text) - if(cl) cl() - assertEquals(stdout.trim(), shell.exec(*:commands, res: "stdout")) - if(cl) cl() - assertEquals(stdout.getBytes("UTF-8"), shell.exec(*:commands, res: "stdoutBytes")) - if(cl) cl() - assertEquals(stderr.trim(), shell.exec(*:commands, res: "stderr")) - if(cl) cl() - assertEquals(stderr.getBytes("UTF-8"), shell.exec(*:commands, res: "stderrBytes")) - if(cl) cl() - assertEquals([exitValue: exitValue, stdout: stdout.trim(), stderr: stderr.trim()], - shell.exec(*:commands, res: "all")) - if(cl) cl() - def res = shell.exec(*:commands, res: "allBytes") - assertEquals(exitValue, res.exitValue) - assertEquals(stdout.getBytes("UTF8"), res.stdout) - assertEquals(stderr.getBytes("UTF8"), res.stderr) - if(cl) cl() - } - - void testTail() - { - FileSystemImpl.createTempFileSystem() { FileSystem fs -> - def shell = new ShellImpl(fileSystem: fs) - - def content = new StringBuilder() - - def f = fs.withOutputStream('testFile') { file, out -> - (1..1000).each { lineNumber -> - def line = "this is line ${lineNumber}\n" - out.write(line.getBytes('UTF-8')) - content = content << line.toString() - } - return file - } - - assertEquals("""this is line 997 -this is line 998 -this is line 999 -this is line 1000 -""", shell.tail(f, 4).text) - - assertEquals(content.toString(), shell.tail(f, -1).text) - - assertNull(shell.tail(location: 'do not exists')) } } @@ -407,196 +84,4 @@ this is line 1000 assertNull(shell.env.storePassword) // should be filtered out } } - - /** - * Test that untar handles gzip properly - */ - void testUntar() - { - FileSystemImpl.createTempFileSystem() { FileSystem fs -> - def shell = new ShellImpl(fileSystem: fs) - - ["./src/test/resources/testUntar_tar", "./src/test/resources/testUntar_tgz"].each { file -> - def fetchedFile = new File(file).canonicalFile.toURI() - - def untarred = shell.untar(shell.fetch(fetchedFile)) - - assertEquals("for ${file}", ['a.txt', 'b.txt', 'c.txt'], shell.ls(untarred).filename.sort()) - } - } - } - - /** - * Test the grep capability - */ - void testGrep() - { - FileSystemImpl.createTempFileSystem() { FileSystem fs -> - def shell = new ShellImpl(fileSystem: fs) - - Resource tempFile = fs.tempFile() - fs.saveContent(tempFile, """line 1 abc -line 2 def -line 3 abcdef -""") - - assertEquals(['line 1 abc', 'line 3 abcdef'], shell.grep(tempFile, /abc/)) - assertEquals(2, shell.grep(tempFile, /abc/, [count: true])) - assertEquals(['line 1 abc'], shell.grep(tempFile, /abc/, [maxCount: 1])) - assertEquals(1, shell.grep(tempFile, /abc/, [count: true, maxCount: 1])) - - // test for 'out' option - def sw = new StringWriter() - assertEquals(sw, shell.grep(tempFile, /abc/, [out: sw])) - assertEquals('line 1 abcline 3 abcdef', sw.toString()) - } - } - - - /** - * Test for listening capability - */ - void testListening() - { - def shell = new ShellImpl() - assertFalse(shell.listening('localhost', 60000)) - - def serverSocket = new ServerSocket(60000) - - def thread = Thread.startDaemon { - try - { - while(true) - { - serverSocket.accept { socket -> - socket.close() - } - } - } - catch (InterruptedIOException e) - { - if(log.isDebugEnabled()) - log.debug("ok because the thread is interrupted... ignored", e) - } - } - - assertTrue(shell.listening('localhost', 60000)) - thread.interrupt() - thread.join(1000) - } - - void testGzip() - { - FileSystemImpl.createTempFileSystem() { FileSystem fs -> - def shell = new ShellImpl(fileSystem: fs) - - def root = shell.toResource('/root') - - shell.saveContent('/root/dir1/a.txt', 'this is a test aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') - shell.saveContent('/root/dir1/b.txt', 'this is a test baaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') - shell.saveContent('/root/dir1/dir2/c.txt', 'this is a test caaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') - shell.saveContent('/root/dir1/dir2/d.txt', 'this is a test daaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') - shell.saveContent('/root/dir1/dir2/e.txt', 'e') // this will generate a negative delta! - - def expectedResult = leavesPaths(root) - def originalSizes = GroovyCollectionsUtils.toMapKey(expectedResult) { shell.toResource(it).size() } - - assertEquals('/root/dir1/dir2/d.txt.gz', shell.gzip('/root/dir1/dir2/d.txt').path) - - expectedResult << '/root/dir1/dir2/d.txt.gz' - expectedResult.remove('/root/dir1/dir2/d.txt') - - assertTrue(GroovyCollectionsUtils.compareIgnoreType(expectedResult, leavesPaths(root))) - assertEquals('this is a test daaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', shell.gunzip('/root/dir1/dir2/d.txt.gz', '/gunzip/d.txt').file.text) - - assertEquals('/root/dir1/dir2/d.txt', shell.gunzip('/root/dir1/dir2/d.txt.gz').path) - expectedResult << '/root/dir1/dir2/d.txt' - expectedResult.remove('/root/dir1/dir2/d.txt.gz') - assertTrue(GroovyCollectionsUtils.compareIgnoreType(expectedResult, leavesPaths(root))) - - assertEquals('this is a test daaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', shell.readContent('/root/dir1/dir2/d.txt')) - - // not recursive => no changes - assertEquals([:], shell.gzipFileOrFolder('/root', false)) - assertTrue(GroovyCollectionsUtils.compareIgnoreType(expectedResult, leavesPaths(root))) - - def res = shell.gzipFileOrFolder('/root', true) - expectedResult = ['/root/dir1/a.txt.gz', '/root/dir1/b.txt.gz', '/root/dir1/dir2/c.txt.gz', '/root/dir1/dir2/d.txt.gz', '/root/dir1/dir2/e.txt.gz'] - def sizes = GroovyCollectionsUtils.toMapKey(expectedResult) { shell.toResource(it).size() } - - sizes.each { path, size -> - assertEquals(originalSizes[path - '.gz'] - size, res[shell.toResource(path)]) - } - - assertEquals(expectedResult, res.keySet().path.sort()) - assertEquals(expectedResult, leavesPaths(root).toArray().sort()) - } - } - - void testRecurse() - { - FileSystemImpl.createTempFileSystem() { FileSystem fs -> - def shell = new ShellImpl(fileSystem: fs) - - def root = shell.toResource('/root') - shell.saveContent('/root/dir1/a.txt', 'this is a test a') - shell.saveContent('/root/dir1/b.txt', 'this is a test b') - shell.saveContent('/root/dir1/dir2/c.txt', 'this is a test c') - shell.saveContent('/root/dir1/dir2/d.txt', 'this is a test d') - shell.saveContent('/root/e.txt', 'e') - - // every file and dir under root - def files = [] - fs.eachChildRecurse(root) { r -> - files << r.path - } - files.sort() - - def expectedFiles = ['/root/dir1', '/root/e.txt', '/root/dir1/a.txt', '/root/dir1/b.txt', '/root/dir1/dir2', '/root/dir1/dir2/c.txt', '/root/dir1/dir2/d.txt'].sort() - assertEquals(expectedFiles, files) - - // find only dirs under root - def dirs = fs.findAll('/root') { r -> - if (r.isDirectory()) { - return true - } else { - return false - } - } - - def expectedDirs = ['/root/dir1', '/root/dir1/dir2'] - assertEquals(expectedDirs, dirs.collect {it.path}) - - // make everything non-writable - shell.chmodRecursive(root, "u-w") - def writableFiles = files.findAll { shell.toResource(it).file.canWrite() } - assertEquals([], writableFiles) - - // make it writable now - shell.chmodRecursive(root, "u+w") - writableFiles = files.findAll { shell.toResource(it).file.canWrite() } - assertEquals(files, writableFiles) - } - } - - void testReplaceTokens() - { - FileSystemImpl.createTempFileSystem() { FileSystem fs -> - def shell = new ShellImpl(fileSystem: fs) - - assertEquals('abc foo efg bar hij foo', - shell.replaceTokens('abc @token1@ efg @token2@ hij @token1@', - [token1: 'foo', token2: 'bar'])) - - Resource testFile = shell.saveContent('test.txt', 'abc @token1@ efg @token2@ hij @token1@', - [token1: 'foo', token2: 'bar']) - - assertEquals('abc foo efg bar hij foo', shell.fetchContent(testFile)) - } - } - - private def leavesPaths(Resource root) - { - new TreeSet(GroovyIOUtils.findAll(root) { !it.isDirectory() }.collect { it.path }) - } } diff --git a/agent/org.linkedin.glu.agent-rest-common/src/main/groovy/org/linkedin/glu/agent/rest/common/AgentRestUtils.groovy b/agent/org.linkedin.glu.agent-rest-common/src/main/groovy/org/linkedin/glu/agent/rest/common/AgentRestUtils.groovy index d5bf4768..ca623184 100644 --- a/agent/org.linkedin.glu.agent-rest-common/src/main/groovy/org/linkedin/glu/agent/rest/common/AgentRestUtils.groovy +++ b/agent/org.linkedin.glu.agent-rest-common/src/main/groovy/org/linkedin/glu/agent/rest/common/AgentRestUtils.groovy @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012 Yan Pujante + * Copyright (c) 2012-2013 Yan Pujante * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -16,16 +16,12 @@ package org.linkedin.glu.agent.rest.common +import org.linkedin.glu.agent.api.AgentException +import org.linkedin.glu.groovy.utils.rest.GluGroovyRestUtils import org.linkedin.groovy.util.rest.RestException import org.restlet.data.Status -import org.linkedin.glu.agent.api.AgentException -import org.linkedin.glu.utils.io.NullOutputStream -import org.linkedin.glu.utils.io.MultiplexedInputStream import org.slf4j.Logger import org.slf4j.LoggerFactory -import org.linkedin.glu.groovy.utils.rest.GluGroovyRestUtils -import org.linkedin.glu.commands.impl.StreamType -import org.linkedin.glu.groovy.utils.json.GluGroovyJsonUtils /** * @author yan@pongasoft.com */ @@ -62,65 +58,4 @@ public class AgentRestUtils { GluGroovyRestUtils.rebuildException(restException) } - - /** - * Demultiplexes the exec stream as generated by - * {@link org.linkedin.glu.agent.api.Shell#exec(Map)} when args.res is - * stream. The following is equivalent: - * - * OutputStream myStdout = ... - * OutputStream myStderr = ... - * - * exec(command: xxx, stdout: myStdout, stderr: myStderr, res: exitValue) - * - * is 100% equivalent to: - * - * demultiplexExecStream(exec(command: xxx, res: stream), myStdout, myStderr) - * - * @param execStream the stream as generated by {@link org.linkedin.glu.agent.api.Shell#exec(Map)} - * @param stdout the stream to write the output (optional, can be null) - * @param stderr the stream to write the error (optional, can be null) - * @return the value returned by the executed subprocess - * @throws org.linkedin.glu.agent.api.ShellExecException when there was an error executing the - * shell script and args.failOnError was set to true - */ - public static def demultiplexExecStream(InputStream execStream, - OutputStream stdout, - OutputStream stderr) - { - def exitValueStream = new ByteArrayOutputStream() - def exitErrorStream = new ByteArrayOutputStream() - - def streams = [:] - - streams[StreamType.stdout.multiplexName] = stdout ?: NullOutputStream.INSTANCE - streams[StreamType.stderr.multiplexName] = stderr ?: NullOutputStream.INSTANCE - streams[StreamType.exitValue.multiplexName] = exitValueStream - streams[StreamType.exitError.multiplexName] = exitErrorStream - - // we demultiplex the stream - MultiplexedInputStream.demultiplex(execStream, streams) - - // it means we got an exception, we throw it back - if(exitErrorStream.size() > 0) - { - throw GluGroovyJsonUtils.rebuildException(new String(exitErrorStream.toByteArray(), "UTF-8")) - } - else - { - if(exitValueStream.size() == 0) - return null - - String exitValueAsString = new String(exitValueStream.toByteArray(), "UTF-8") - try - { - return Integer.valueOf(exitValueAsString) - } - catch(NumberFormatException e) - { - // this should not really happen but just in case... - return exitValueAsString - } - } - } } \ No newline at end of file diff --git a/commands/org.linkedin.glu.commands-impl/src/main/groovy/org/linkedin/glu/commands/impl/AbstractCommandStreamStorage.groovy b/commands/org.linkedin.glu.commands-impl/src/main/groovy/org/linkedin/glu/commands/impl/AbstractCommandStreamStorage.groovy index 3ed68d05..ff63ec80 100644 --- a/commands/org.linkedin.glu.commands-impl/src/main/groovy/org/linkedin/glu/commands/impl/AbstractCommandStreamStorage.groovy +++ b/commands/org.linkedin.glu.commands-impl/src/main/groovy/org/linkedin/glu/commands/impl/AbstractCommandStreamStorage.groovy @@ -17,6 +17,7 @@ package org.linkedin.glu.commands.impl import org.linkedin.glu.groovy.utils.collections.GluGroovyCollectionUtils +import org.linkedin.glu.groovy.utils.io.StreamType import org.linkedin.groovy.util.config.Config import java.util.concurrent.TimeoutException import org.linkedin.glu.groovy.utils.io.InputGeneratorStream diff --git a/commands/org.linkedin.glu.commands-impl/src/main/groovy/org/linkedin/glu/commands/impl/CommandStreamStorage.groovy b/commands/org.linkedin.glu.commands-impl/src/main/groovy/org/linkedin/glu/commands/impl/CommandStreamStorage.groovy index be0a8005..e6abae2c 100644 --- a/commands/org.linkedin.glu.commands-impl/src/main/groovy/org/linkedin/glu/commands/impl/CommandStreamStorage.groovy +++ b/commands/org.linkedin.glu.commands-impl/src/main/groovy/org/linkedin/glu/commands/impl/CommandStreamStorage.groovy @@ -17,6 +17,7 @@ package org.linkedin.glu.commands.impl import org.linkedin.glu.groovy.utils.concurrent.FutureTaskExecution +import org.linkedin.glu.groovy.utils.io.StreamType import org.linkedin.glu.utils.concurrent.Submitter /** diff --git a/commands/org.linkedin.glu.commands-impl/src/main/groovy/org/linkedin/glu/commands/impl/FileSystemCommandExecutionIOStorage.groovy b/commands/org.linkedin.glu.commands-impl/src/main/groovy/org/linkedin/glu/commands/impl/FileSystemCommandExecutionIOStorage.groovy index 89fe5c2c..642fe370 100644 --- a/commands/org.linkedin.glu.commands-impl/src/main/groovy/org/linkedin/glu/commands/impl/FileSystemCommandExecutionIOStorage.groovy +++ b/commands/org.linkedin.glu.commands-impl/src/main/groovy/org/linkedin/glu/commands/impl/FileSystemCommandExecutionIOStorage.groovy @@ -16,6 +16,7 @@ package org.linkedin.glu.commands.impl +import org.linkedin.glu.groovy.utils.io.StreamType import org.linkedin.groovy.util.io.fs.FileSystem import org.linkedin.groovy.util.json.JsonUtils import org.linkedin.util.annotations.Initializable diff --git a/commands/org.linkedin.glu.commands-impl/src/main/groovy/org/linkedin/glu/commands/impl/FileSystemStreamStorage.groovy b/commands/org.linkedin.glu.commands-impl/src/main/groovy/org/linkedin/glu/commands/impl/FileSystemStreamStorage.groovy index 661638e7..d81f39d8 100644 --- a/commands/org.linkedin.glu.commands-impl/src/main/groovy/org/linkedin/glu/commands/impl/FileSystemStreamStorage.groovy +++ b/commands/org.linkedin.glu.commands-impl/src/main/groovy/org/linkedin/glu/commands/impl/FileSystemStreamStorage.groovy @@ -16,6 +16,7 @@ package org.linkedin.glu.commands.impl +import org.linkedin.glu.groovy.utils.io.StreamType import org.linkedin.util.io.resource.Resource import org.linkedin.groovy.util.collections.GroovyCollectionsUtils diff --git a/commands/org.linkedin.glu.commands-impl/src/main/groovy/org/linkedin/glu/commands/impl/MemoryStreamStorage.groovy b/commands/org.linkedin.glu.commands-impl/src/main/groovy/org/linkedin/glu/commands/impl/MemoryStreamStorage.groovy index 4a113e6c..ac82b08e 100644 --- a/commands/org.linkedin.glu.commands-impl/src/main/groovy/org/linkedin/glu/commands/impl/MemoryStreamStorage.groovy +++ b/commands/org.linkedin.glu.commands-impl/src/main/groovy/org/linkedin/glu/commands/impl/MemoryStreamStorage.groovy @@ -16,6 +16,8 @@ package org.linkedin.glu.commands.impl +import org.linkedin.glu.groovy.utils.io.StreamType + /** * @author yan@pongasoft.com */ class MemoryStreamStorage extends AbstractCommandStreamStorage diff --git a/commands/org.linkedin.glu.commands-impl/src/main/groovy/org/linkedin/glu/commands/impl/NoCommandStreamStorage.groovy b/commands/org.linkedin.glu.commands-impl/src/main/groovy/org/linkedin/glu/commands/impl/NoCommandStreamStorage.groovy index e87f738b..0209f687 100644 --- a/commands/org.linkedin.glu.commands-impl/src/main/groovy/org/linkedin/glu/commands/impl/NoCommandStreamStorage.groovy +++ b/commands/org.linkedin.glu.commands-impl/src/main/groovy/org/linkedin/glu/commands/impl/NoCommandStreamStorage.groovy @@ -16,6 +16,8 @@ package org.linkedin.glu.commands.impl +import org.linkedin.glu.groovy.utils.io.StreamType + /** * @author yan@pongasoft.com */ public class NoCommandStreamStorage extends AbstractCommandStreamStorage diff --git a/commands/org.linkedin.glu.commands-impl/src/test/groovy/test/commands/impl/TestFileSystemCommandExecutionIOStorage.groovy b/commands/org.linkedin.glu.commands-impl/src/test/groovy/test/commands/impl/TestFileSystemCommandExecutionIOStorage.groovy index 552d7d68..44fa41af 100644 --- a/commands/org.linkedin.glu.commands-impl/src/test/groovy/test/commands/impl/TestFileSystemCommandExecutionIOStorage.groovy +++ b/commands/org.linkedin.glu.commands-impl/src/test/groovy/test/commands/impl/TestFileSystemCommandExecutionIOStorage.groovy @@ -24,7 +24,7 @@ import org.linkedin.glu.commands.impl.GluCommandFactory import java.text.SimpleDateFormat import org.linkedin.groovy.util.json.JsonUtils import org.linkedin.glu.commands.impl.CommandStreamStorage -import org.linkedin.glu.commands.impl.StreamType +import org.linkedin.glu.groovy.utils.io.StreamType import org.linkedin.util.clock.Timespan import org.linkedin.util.io.resource.Resource import org.linkedin.glu.commands.impl.CommandExecution @@ -32,7 +32,6 @@ import org.linkedin.glu.utils.io.MultiplexedInputStream import org.linkedin.glu.groovy.utils.json.GluGroovyJsonUtils import org.linkedin.glu.groovy.utils.io.GluGroovyIOUtils import org.linkedin.glu.groovy.utils.plugins.PluginServiceImpl -import org.linkedin.groovy.util.collections.IgnoreTypeComparator import org.linkedin.glu.groovy.utils.test.GluGroovyTestUtils /** diff --git a/commands/org.linkedin.glu.commands-impl/src/test/groovy/test/commands/impl/TestMemoryCommandExecutionIOStorage.groovy b/commands/org.linkedin.glu.commands-impl/src/test/groovy/test/commands/impl/TestMemoryCommandExecutionIOStorage.groovy index 961299f3..b8ec10d9 100644 --- a/commands/org.linkedin.glu.commands-impl/src/test/groovy/test/commands/impl/TestMemoryCommandExecutionIOStorage.groovy +++ b/commands/org.linkedin.glu.commands-impl/src/test/groovy/test/commands/impl/TestMemoryCommandExecutionIOStorage.groovy @@ -21,7 +21,7 @@ import org.linkedin.glu.commands.impl.CommandExecution import org.linkedin.glu.commands.impl.CommandStreamStorage import org.linkedin.glu.commands.impl.GluCommandFactory import org.linkedin.glu.commands.impl.MemoryCommandExecutionIOStorage -import org.linkedin.glu.commands.impl.StreamType +import org.linkedin.glu.groovy.utils.io.StreamType import org.linkedin.groovy.util.collections.GroovyCollectionsUtils import org.linkedin.groovy.util.json.JsonUtils import org.linkedin.util.clock.SettableClock diff --git a/orchestration/org.linkedin.glu.orchestration-engine/src/main/groovy/org/linkedin/glu/orchestration/engine/commands/CommandExecutionStream.groovy b/orchestration/org.linkedin.glu.orchestration-engine/src/main/groovy/org/linkedin/glu/orchestration/engine/commands/CommandExecutionStream.groovy index 3c662445..36042252 100644 --- a/orchestration/org.linkedin.glu.orchestration-engine/src/main/groovy/org/linkedin/glu/orchestration/engine/commands/CommandExecutionStream.groovy +++ b/orchestration/org.linkedin.glu.orchestration-engine/src/main/groovy/org/linkedin/glu/orchestration/engine/commands/CommandExecutionStream.groovy @@ -16,7 +16,7 @@ package org.linkedin.glu.orchestration.engine.commands -import org.linkedin.glu.commands.impl.StreamType +import org.linkedin.glu.groovy.utils.io.StreamType import org.linkedin.glu.commands.impl.CommandStreamStorage import org.linkedin.glu.utils.io.LimitedOutputStream import org.apache.tools.ant.util.TeeOutputStream diff --git a/orchestration/org.linkedin.glu.orchestration-engine/src/main/groovy/org/linkedin/glu/orchestration/engine/commands/CommandsServiceImpl.groovy b/orchestration/org.linkedin.glu.orchestration-engine/src/main/groovy/org/linkedin/glu/orchestration/engine/commands/CommandsServiceImpl.groovy index bcda4b04..69800089 100644 --- a/orchestration/org.linkedin.glu.orchestration-engine/src/main/groovy/org/linkedin/glu/orchestration/engine/commands/CommandsServiceImpl.groovy +++ b/orchestration/org.linkedin.glu.orchestration-engine/src/main/groovy/org/linkedin/glu/orchestration/engine/commands/CommandsServiceImpl.groovy @@ -35,7 +35,7 @@ import org.slf4j.LoggerFactory import java.util.concurrent.TimeoutException import org.linkedin.glu.groovy.utils.collections.GluGroovyCollectionUtils -import org.linkedin.glu.commands.impl.StreamType +import org.linkedin.glu.groovy.utils.io.StreamType import org.linkedin.glu.commands.impl.CommandExecution import org.linkedin.glu.commands.impl.CommandExecutionIOStorage import org.linkedin.glu.commands.impl.CommandStreamStorage diff --git a/orchestration/org.linkedin.glu.orchestration-engine/src/test/groovy/test/orchestration/engine/commands/TestCommandsServiceImpl.groovy b/orchestration/org.linkedin.glu.orchestration-engine/src/test/groovy/test/orchestration/engine/commands/TestCommandsServiceImpl.groovy index 8f60585e..f8fac821 100644 --- a/orchestration/org.linkedin.glu.orchestration-engine/src/test/groovy/test/orchestration/engine/commands/TestCommandsServiceImpl.groovy +++ b/orchestration/org.linkedin.glu.orchestration-engine/src/test/groovy/test/orchestration/engine/commands/TestCommandsServiceImpl.groovy @@ -18,7 +18,7 @@ package test.orchestration.engine.commands import groovy.mock.interceptor.MockFor import org.linkedin.glu.commands.impl.MemoryCommandExecutionIOStorage -import org.linkedin.glu.commands.impl.StreamType +import org.linkedin.glu.groovy.utils.io.StreamType import org.linkedin.glu.groovy.utils.io.InputGeneratorStream import org.linkedin.glu.orchestration.engine.agents.AgentsService diff --git a/utils/org.linkedin.glu.utils/build.gradle b/utils/org.linkedin.glu.utils/build.gradle index a868872e..02d6e75c 100644 --- a/utils/org.linkedin.glu.utils/build.gradle +++ b/utils/org.linkedin.glu.utils/build.gradle @@ -20,10 +20,13 @@ apply plugin: 'org.linkedin.release' dependencies { compile spec.external.utilsMiscCore compile spec.external.utilsMiscGroovy + compile spec.external.mimeUtil compile spec.external.groovy testCompile spec.external.junit runtime spec.external.slf4jLog4j runtime spec.external.log4j + + testRuntime spec.external.ivy } diff --git a/utils/org.linkedin.glu.utils/src/main/groovy/org/linkedin/glu/groovy/utils/io/GluGroovyIOUtils.groovy b/utils/org.linkedin.glu.utils/src/main/groovy/org/linkedin/glu/groovy/utils/io/GluGroovyIOUtils.groovy index 32c3222b..fc80a7a5 100644 --- a/utils/org.linkedin.glu.utils/src/main/groovy/org/linkedin/glu/groovy/utils/io/GluGroovyIOUtils.groovy +++ b/utils/org.linkedin.glu.utils/src/main/groovy/org/linkedin/glu/groovy/utils/io/GluGroovyIOUtils.groovy @@ -18,6 +18,9 @@ package org.linkedin.glu.groovy.utils.io import org.codehaus.groovy.control.CompilationUnit import org.codehaus.groovy.control.CompilerConfiguration +import org.linkedin.glu.groovy.utils.json.GluGroovyJsonUtils +import org.linkedin.glu.utils.io.MultiplexedInputStream +import org.linkedin.glu.utils.io.NullOutputStream import org.linkedin.groovy.util.ant.AntUtils import org.linkedin.groovy.util.io.GroovyIOUtils import org.linkedin.util.io.resource.Resource @@ -93,4 +96,64 @@ public class GluGroovyIOUtils extends GroovyIOUtils cipher.init(mode, key) return cipher } + + /** + * Demultiplexes the exec stream as generated by + * {@link org.linkedin.glu.groovy.utils.shell.Shell#exec(Map)} when args.res is + * stream. The following is equivalent: + * + * OutputStream myStdout = ... + * OutputStream myStderr = ... + * + * exec(command: xxx, stdout: myStdout, stderr: myStderr, res: exitValue) + * + * is 100% equivalent to: + * + * demultiplexExecStream(exec(command: xxx, res: stream), myStdout, myStderr) + * + * @param execStream the stream as generated by {@link org.linkedin.glu.groovy.utils.shell.Shell#exec(Map)} + * @param stdout the stream to write the output (optional, can be null) + * @param stderr the stream to write the error (optional, can be null) + * @return the value returned by the executed sub-process + */ + public static def demultiplexExecStream(InputStream execStream, + OutputStream stdout, + OutputStream stderr) + { + def exitValueStream = new ByteArrayOutputStream() + def exitErrorStream = new ByteArrayOutputStream() + + def streams = [:] + + streams[StreamType.stdout.multiplexName] = stdout ?: NullOutputStream.INSTANCE + streams[StreamType.stderr.multiplexName] = stderr ?: NullOutputStream.INSTANCE + streams[StreamType.exitValue.multiplexName] = exitValueStream + streams[StreamType.exitError.multiplexName] = exitErrorStream + + // we demultiplex the stream + MultiplexedInputStream.demultiplex(execStream, streams) + + // it means we got an exception, we throw it back + if(exitErrorStream.size() > 0) + { + throw GluGroovyJsonUtils.rebuildException(new String(exitErrorStream.toByteArray(), "UTF-8")) + } + else + { + if(exitValueStream.size() == 0) + return null + + String exitValueAsString = new String(exitValueStream.toByteArray(), "UTF-8") + try + { + return Integer.valueOf(exitValueAsString) + } + catch(NumberFormatException e) + { + // this should not really happen but just in case... + return exitValueAsString + } + } + } + } \ No newline at end of file diff --git a/commands/org.linkedin.glu.commands-impl/src/main/groovy/org/linkedin/glu/commands/impl/StreamType.groovy b/utils/org.linkedin.glu.utils/src/main/groovy/org/linkedin/glu/groovy/utils/io/StreamType.groovy similarity index 91% rename from commands/org.linkedin.glu.commands-impl/src/main/groovy/org/linkedin/glu/commands/impl/StreamType.groovy rename to utils/org.linkedin.glu.utils/src/main/groovy/org/linkedin/glu/groovy/utils/io/StreamType.groovy index c50f4f8e..e4610c35 100644 --- a/commands/org.linkedin.glu.commands-impl/src/main/groovy/org/linkedin/glu/commands/impl/StreamType.groovy +++ b/utils/org.linkedin.glu.utils/src/main/groovy/org/linkedin/glu/groovy/utils/io/StreamType.groovy @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012 Yan Pujante + * Copyright (c) 2012-2013 Yan Pujante * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -14,7 +14,7 @@ * the License. */ -package org.linkedin.glu.commands.impl +package org.linkedin.glu.groovy.utils.io /** * @author yan@pongasoft.com */ diff --git a/utils/org.linkedin.glu.utils/src/main/groovy/org/linkedin/glu/groovy/utils/shell/Shell.groovy b/utils/org.linkedin.glu.utils/src/main/groovy/org/linkedin/glu/groovy/utils/shell/Shell.groovy new file mode 100644 index 00000000..06f41996 --- /dev/null +++ b/utils/org.linkedin.glu.utils/src/main/groovy/org/linkedin/glu/groovy/utils/shell/Shell.groovy @@ -0,0 +1,500 @@ +/* + * Copyright (c) 2010-2010 LinkedIn, Inc + * Portions Copyright (c) 2011-2013 Yan Pujante + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.linkedin.glu.groovy.utils.shell + +import org.linkedin.util.io.resource.Resource +import org.linkedin.groovy.util.io.fs.FileSystem + +/** + * Contains shell related methods. + */ +def interface Shell extends FileSystem +{ + /** + * Try to guess the mime types of a given file + * + * @param file ({@see #toResource(Object)} for possible values) + * @return a list of mime types (strings) + */ + Collection getMimeTypes(file) + + /** + * Unzips the provided file in a temporary location + * + * @param file ({@see #toResource(Object)} for possible values) + * @return the location of the temporary location (as a Resource) + */ + Resource unzip(file) + + /** + * Unzips the provided file in the provided location + * + * @param file ({@see #toResource(Object)} for possible values) + * @param toDir ({@see #toResource(Object)} for possible values) + * @return toDir (as a Resource) + */ + Resource unzip(file, toDir) + + /** + * Untars the provided file in a temporary location. Note that the implementation will try + * to detect if the file is also gziped and unzip it first (equivalent to tar -zxf) + * + * @param file ({@see #toResource(Object)} for possible values) + * @return the location of the temporary location (as a Resource) + */ + Resource untar(file) + + /** + * Untars the provided file in the provided location. Note that the implementation will try + * to detect if the file is also gziped and unzip it first (equivalent to tar -zxf) + * + * @param file ({@see #toResource(Object)} for possible values) + * @return toDir (as a Resource) + */ + Resource untar(file, toDir) + + /** + * Gunzips the provided file in a temporary location + * + * @param file ({@see #toResource(Object)} for possible values) + * @return the location of the temporary location (as a Resource) + */ + Resource gunzip(file) + + /** + * Gunzips the provided file in the provided location + * + * @param file ({@see #toResource(Object)} for possible values) + * @param toDir ({@see #toResource(Object)} for possible values) + * @return toDir (as a Resource) + */ + Resource gunzip(file, toFile) + + /** + * Compresses the provided file (in the same folder) + * + * @param file ({@see #toResource(Object)} for possible values) + * @return the gziped file or the original file if not zipped (as a Resource) + */ + Resource gzip(file) + + /** + * Compresses the provided file as toFile + * + * @param file ({@see #toResource(Object)} for possible values) + * @param toFile ({@see #toResource(Object)} for possible values) + * @return toFile (as a Resource) + */ + Resource gzip(file, toFile) + + /** + * Compresses each file in a folder + * + * @param fileOrFolder a file (behaves like {@link #gzip(Object)}) or a folder + * ({@see #toResource(Object)} for possible values) + * @param recurse if true then recursively process all folders + * @return a map with key is resource and value is delta in size (original - new) (note that it + * contains only the resources that are modified!) + */ + Map gzipFileOrFolder(fileOrFolder, boolean recurse) + + /** + * untar + decrypt using the encryption keys (encrytion keys are automatically provided by glu + * and are available in any glu script with args.encriptionKeys) + * + * @param file ({@see #toResource(Object)} for possible values) + * @param toDir ({@see #toResource(Object)} for possible values) + * @param encryptionKeys + * @return toDir (as a Resource) + */ + Resource untarAndDecrypt(file, toDir, encryptionKeys) + + /** + * Exporting ant access to the shell to run any ant command. + * + * @return whatever the closure returns + */ + def ant(Closure closure) + + /** + * Fetches the file pointed to by the location. If the location is already a {@link File} then + * simply returns it. The location can be a String or URI and must + * contain a scheme. Example of locations: http://locahost:8080/file.txt', + * file:/tmp/file.txt, ivy:/org.linkedin/util-core/1.0.0. + * + * @param location the location you want to fetch (usually remote) + * @return where the location was fetched (locally) (as a Resource) + */ + Resource fetch(location) + + /** + * Fetches the file pointed to by the location. The location can be File, + * a String or URI and must contain a scheme. Example of locations: + * http://locahost:8080/file.txt', file:/tmp/file.txt, + * ivy:/org.linkedin/util-core/1.0.0. The difference with the other fetch method + * is that it fetches the file in the provided destination rather than in the tmp space. + * + * @param location the location you want to fetch (usually remote) + * @param destination ({@see #toResource(Object)} for possible values) + * @return where the location was fetched (locally) (as a Resource) (if + * destination is a file then returns destination as a + * Resource otherwise it will be a Resource inside the + * destination folder. + */ + Resource fetch(location, destination) + + /** + * Returns the content of the location as a String or + * null if the location is not reachable + * @param location the location you want to fetch (usually remote) + * @return the content as a Strint or null + */ + String cat(location) + + /** + * Issue a 'HEAD' request. The location should be an http or https link. + * + * @param location + * @return a map with the following entries: + * responseCode: 200, 404... {@link java.net.HttpURLConnection#getResponseCode()} + * responseMessage: message {@link java.net.HttpURLConnection#getResponseMessage()} + * headers: representing all the headers {@link java.net.URLConnection#getHeaderFields()} + */ + Map httpHead(location) + + /** + * Issue a 'POST' request. The location should be an http or https link. The request will be + * made with application/x-www-form-urlencoded content type. + * + * @param location + * @param parameters the parameters of the post as map of key value pairs (value can be a single + * value or a collection of values) + * @return a map with the following entries: + * responseCode: 200, 404... {@link java.net.HttpURLConnection#getResponseCode()} + * responseMessage: message {@link java.net.HttpURLConnection#getResponseMessage()} + * headers: representing all the headers {@link java.net.URLConnection#getHeaderFields()} + */ + Map httpPost(location, Map parameters) + + /** + * Similarly to the unix grep command, checks the location one line at a time and returns + * all the lines which matches the pattern. + * + * @param location (see {@link #fetch(Object)} for possible values) + * @param pattern regular expression (see java pattern) + * @return a list of strings + */ + Collection grep(location, pattern) + + /** + * Similarly to the unix grep command, checks the location one line at a time and returns + * all the lines which matches the pattern + * + * @param location (see {@link #fetch(Object)} for possible values) + * @param pattern regular expression (see java pattern) + * @param options options to the command: + * out: an object to output the lines which match (default to []) + * count: returns the count only (does not use out) + * maxCount: stop reading after maxCount matches + * @return depends on options + */ + def grep(location, pattern, options) + + /** + * Executes a shell command... the command will be delegated straight to shell and the output of + * the shell command is returned. + * + * @param executable ({@see #toResource(Object)} for possible values) + * @param executableArgs either a String or a Collection + * @return whatever output the shell command returns + */ + String exec(executable, executableArgs) + + /** + * Executes a shell command... the command will be delegated straight to shell and the output of + * the shell command is returned. + */ + /** + * Executes a shell command... the command will be delegated straight to shell and the output of + * the shell command is returned. + * + * @param executable ({@see #toResource(Object)} for possible values) + * @param executableArgs as a vararg of executable args + * @return whatever output the shell command returns + */ + String exec(executable, Object... executableArgs) + + /** + * Executes a shell command... the command will be delegated straight to shell and the output of + * the shell command is returned. + * + * @param command the full command (including the arguments) as a String + * @return whatever output the shell command returns + */ + String exec(String command) + + /** + * Executes a shell command... the command will be delegated straight to shell and the output of + * the shell command is returned. + * + * @param command the full command (including the arguments) as a Collection + * @return whatever output the shell command returns + */ + String exec(Collection command) + + /** + * More generic form of the exec call which allows you to configure what you provide and what you + * expect. This call is blocking (unless you request a stream for the result). + * + * - Note that if you provide a closure for args.stdout or + * args.stderr it will be executed in a separate thread (in order to avoid + * blocking indefinitely). + * + * - Note that if you request stdout, stderr or + * all for the result of this call, stdout and stderr are + * converted to a String using (by default) the "UTF-8" charset and all lines + * are terminated with "\n" and the last "\n" is removed. Use stdoutBytes or + * stderrBytes if you wish to get the bytes directly + * + * - Note that if you request stream, then the call return immediately + * (it is non blocking in this case) and you get a single InputStream + * which multiplexes stdout/stderr and exit value (see MultiplexedInputStream for + * details). In this case you should make sure to read the entire stream and properly close it + * as shown in the following code example: + * + * InputStream stream = shell.exec(command: 'xxx', res: 'stream') + * try + * { + * // read stream + * } + * finally + * { + * stream.close() + * } + * + * - Note that if you request stdoutStream or stderrStream, then the + * call return immediately (it is non blocking in this case) and you get the stream you requested. + * In this case you should make sure to read the entire stream and properly close it as + * shown in the following code example: + * + * InputStream stream = shell.exec(command: 'xxx', res: 'stdoutStream') + * try + * { + * // read stream + * } + * finally + * { + * stream.close() + * } + * + * + * @param args.command the command to execute. It will be delegated to the shell so it should + * be native to the OS on which the agent runs (either a String + * or a Collection) (required) + * @param args.pwd the directory from which the command will be run (optional, will + * default to the "current" directory) + * @param args.env a map (String, String) containing environment + * variables to be passed to sub-process. If the value is null, + * it will remove it from the inherited environment variables. (optional) + * @param args.inheritEnv a boolean to determine if the environment variables in effect in the + * agent will be passed down to the sub-process (optional, default to + * true) + * @param args.stdin any input that can "reasonably" be converted into an + * InputStream) to provide to the command line execution + * (optional, default to no stdin) + * @param args.stdout Closure (if you want to handle it yourself), + * OutputStream (where stdout will be written to), + * (optional: depends on args.res) + * @param args.stderr Closure (if you want to handle it yourself), + * OutputStream (where stdout will be written to), + * (optional: depends on args.res) + * @param args.redirectStderr boolean to redirect stderr into stdout + * (optional, default to false). Note that this can also + * be accomplished with the command itself with something like "2>&1" + * @param args.failOnError do you want the command to fail (with an exception) when there is + * an error (default to true) + * @param args.res what do you want the call to return + * stdout, stdoutBytes, stdoutStream + * stderr, stderrBytes, stderrStream + * all, allBytes (a map with 3 parameters, exitValue, stdout, stderr) + * exitValue, + * stream + * (default to stdout) + * @return whatever is specified in args.res + */ + def exec(Map args) + + /** + * Demultiplexes the exec stream as generated by {@link #exec(Map)} when args.res is + * stream. The following is equivalent: + * + * OutputStream myStdout = ... + * OutputStream myStderr = ... + * + * exec(command: xxx, stdout: myStdout, stderr: myStderr, res: exitValue) + * + * is 100% equivalent to: + * + * demultiplexExecStream(exec(command: xxx, res: stream), myStdout, myStderr) + * + * @param execStream the stream as generated by {@link #exec(Map)} + * @param stdout the stream to write the output (optional, can be null) + * @param stderr the stream to write the error (optional, can be null) + * @return the value returned by the executed subprocess + * @throws ShellExecException when there was an error executing the + * shell script and args.failOnError was set to true + */ + def demultiplexExecStream(InputStream execStream, + OutputStream stdout, + OutputStream stderr) + + /** + * Shortcut/More efficient implementation of the more generic {@link #chmod(Object, Object)} call + * + * @param file ({@see #toResource(Object)} for possible values) + * @return file as a Resource + */ + Resource chmodPlusX(file) + + /** + * Changes the permission of the dir and recursively + * + * @param dir ({@see #toResource(Object)} for possible values) + * @param perm expressed in unix fashion (ex: +x) + * @return the {@link Resource} representation of the dir + */ + Resource chmodRecursive(dir, perm) + + /** + * Invokes the closure with an MBeanServerConnection to the jmx control running + * on the vm started with the provided pid. The closure will be invoked with null + * if cannot determine the process. + * + * @param pid the pid of the process you want to get an mbean connection + * @param closure will be called back with a MBeanServerConnection (which will be + * null if cannot connect) + * @return whatever the closure returns + */ + def withMBeanServerConnection(pid, Closure closure) + + /** + * Waits for the condition to be true no longer than the timeout. true + * is returned if the condition was satisfied, false otherwise (if you specify + * noException) + * @param args.timeout how long max to wait (any value convertible to a Timespan) + * @param args.heartbeat how long to wait between calling the condition (any value convertible to + * a Timespan) + * @param args.noException to get false instead of an exception + * @param condition the closure should return true if the condition is satisfied, + * false otherwise + * @return true if condition was satisfied within the timeout, false + * otherwise (if args.noException is provided otherwise an exception will + * be raised) + */ + boolean waitFor(args, Closure condition) + + /** + * Shortcut when no args + * @see #waitFor(Object, Closure) + */ + boolean waitFor(Closure condition) + + /** + * Tail the file + * + * @params file the file to tail ({@see #toResource(Object)} for possible values) + * @params maxLine the number of lines maximum to read + * + * @return the input stream to read the content of the tail + */ + InputStream tail(file, long maxLine) + + /** + * Tail the file + * + * @params args.location the location of the file to tail ({@see #toResource(Object)} for possible values) + * @params args.maxLine the number of lines maximum to read (-1 for the entire file) + * @params args.maxSize the maximum size to read (-1 for the entire file) + * + * @return the input stream to read the content of the tail + */ + InputStream tail(args) + + /** + * @return true if there is a socket open on the server/port combination + */ + boolean listening(server, port) + + /** + * Replaces the tokens provided in the map in the input. Token replacement is using ant token + * replacement class so in the input, the tokens are surrounded by the '@' sign. + * Example: + * + * 
+   * input = "abcd @myToken@"
+   * assert "abcd foo" == replaceTokens(input, [myToken: 'foo'])
+   *
+ */ + String replaceTokens(String input, Map tokens) + + /** + * Processes from through the replacement token mechanism and writes the result to + * to + * + * @param from ({@see #toResource(Object)} for possible values) + * @param to ({@see #toResource(Object)} for possible values)} + * @param tokens a map of token + * @return to as a {@link Resource} + * @see #replaceTokens(String, Map) + */ + Resource replaceTokens(def from, def to, Map tokens) + + /** + * Processes the content to the token replacement method. + * + * @see #saveContent(Object, String) + */ + Resource saveContent(file, String content, Map tokens) + + /** + * Same as withInputStream but wraps in a reader using a configured charset (defaults + * to UTF-8). + * + * @param file ({@see #toResource(Object)} for possible values) + * @return whatever the closure returns + */ + def withReader(file, Closure closure) + + /** + * Same as withOutputStream but wraps in a writer using a configured charset (defaults + * to UTF-8). + * + * @param file ({@see #toResource(Object)} for possible values) + * @return whatever the closure returns + */ + def withWriter(file, Closure closure) + + /** + * Runs the closure in a protected block that will not throw an exception but will return + * null in the case one happens + * + * @return whatever the closure returns unless there is an exception in which case it will + * return null (and log the exception as a debug message) + */ + def noException(Closure closure) +} \ No newline at end of file diff --git a/utils/org.linkedin.glu.utils/src/main/groovy/org/linkedin/glu/groovy/utils/shell/ShellExec.groovy b/utils/org.linkedin.glu.utils/src/main/groovy/org/linkedin/glu/groovy/utils/shell/ShellExec.groovy new file mode 100644 index 00000000..734e0b5e --- /dev/null +++ b/utils/org.linkedin.glu.utils/src/main/groovy/org/linkedin/glu/groovy/utils/shell/ShellExec.groovy @@ -0,0 +1,420 @@ +/* + * Copyright (c) 2012 Yan Pujante + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.linkedin.glu.groovy.utils.shell + +import org.apache.tools.ant.taskdefs.Execute +import org.linkedin.glu.groovy.utils.GluGroovyLangUtils +import org.linkedin.glu.groovy.utils.concurrent.FutureTaskExecution +import org.linkedin.glu.groovy.utils.io.DestroyProcessInputStream +import org.linkedin.glu.groovy.utils.io.InputGeneratorStream +import org.linkedin.glu.groovy.utils.io.StreamType +import org.linkedin.glu.groovy.utils.json.GluGroovyJsonUtils +import org.linkedin.glu.groovy.utils.lang.ProcessWithResult +import org.linkedin.glu.utils.io.EmptyInputStream +import org.linkedin.glu.utils.io.MultiplexedInputStream +import org.linkedin.glu.utils.io.NullOutputStream +import org.linkedin.groovy.util.collections.GroovyCollectionsUtils +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +import java.util.concurrent.ExecutionException + +/** + * Because the logic of exec is quite complicated, it requires its own class + * + * @author yan@pongasoft.com */ +class ShellExec +{ + public static final String MODULE = ShellExec.class.getName(); + public static final Logger log = LoggerFactory.getLogger(MODULE); + + // the shell which created this class + ShellImpl shell + + // the args passed to shell.exec + Map args + + private String _commandLine + private InputStream _stdin + private boolean _redirectStderr + private boolean _failOnError + private def _requiredRes + + // second map contains: stream, streamOfBytes + private Map _processIO = [:] + + // process + private boolean _destroyProcessInFinally = true + private Process _process + + /** + * Use bash -c to run the command line + */ + public static def buildCommandLine(String commandLine) + { + ['bash', '-c', commandLine] + } + + def exec() + { + initCommandLine() + + _stdin = shell.toInputStream(args.stdin) + + _redirectStderr = GluGroovyLangUtils.getOptionalBoolean(args.redirectStderr, false) + _failOnError = GluGroovyLangUtils.getOptionalBoolean(args.failOnError, true) + _requiredRes = args.res ?: 'stdout' + + // stdout + initOutput(StreamType.stdout) + + // stderr + if(!_redirectStderr) + initOutput(StreamType.stderr) + + // builds the process + def pb = new ProcessBuilder(buildCommandLine(_commandLine)) + pb.redirectErrorStream(_redirectStderr) + + // pwd + if(args.pwd) + pb.directory(args.pwd as File) + + // environment + Map environment = pb.environment() + if(!GluGroovyLangUtils.getOptionalBoolean(args.inheritEnv, true)) + environment.clear() + + if(args.env) + { + args.env.each { k, v -> + k = k.toString() + + if(v == null) + environment.remove(k) + else + environment[k] = v.toString() + } + } + + try + { + _process = pb.start() + } + catch(IOException e) + { + _process = new ProcessWithResult(outputStream: NullOutputStream.INSTANCE, + inputStream: EmptyInputStream.INSTANCE, + errorStream: new InputGeneratorStream(e.message), + exitValue: 2) + } + + try + { + afterProcessStarted() + } + finally + { + if(_destroyProcessInFinally) + _process.destroy() + } + } + + private def afterProcessStarted() + { + // stdout + startOutputThread(StreamType.stdout) { _process.inputStream } + + // when redirecting stderr, there is nothing on stderr... no need to create a thread! + if(!_redirectStderr) + { + startOutputThread(StreamType.stderr) { _process.errorStream } + } + + if(_requiredRes.toLowerCase().endsWith("stream")) + return createRequiredInputStream() + else + return executeBlockingCall() + } + + /** + * Creates the appropriate stream but make sure we still destroy the process when the + * stream closes + */ + private InputStream createRequiredInputStream() + { + // execute the block call asynchronously + FutureTaskExecution future = new FutureTaskExecution(executeBlockingCall) + future.description = _commandLine + + InputStream inputStream + + switch(_requiredRes) + { + case "stdoutStream": + inputStream = _process.inputStream + break + + case "stderrStream": + inputStream = _process.errorStream + break + + case "exitValueStream": + inputStream = new InputGeneratorStream({ + try + { + future.get().toString() + } + catch(ExecutionException e) + { + throw e.cause + } + }) + break + + case "stream": + inputStream = createMultiplexedInputStream(future) + break + + default: + throw new RuntimeException("should not be here with ${_requiredRes}") + } + + inputStream = new DestroyProcessInputStream(_process, inputStream) + + // we can no longer destroy the process in the finally, it will be destroyed when the + // input stream is closed + _destroyProcessInFinally = false + + future.runAsync(shell.submitter) + + return inputStream + } + + /** + * Create one input stream which multiples stdout, stderr, exitValue and exitError + */ + private InputStream createMultiplexedInputStream(FutureTaskExecution future) + { + // execute the block call asynchronously + def streams = [:] + + // stdout + if(_processIO[StreamType.stdout]?.stream == null) + { + streams[StreamType.stdout.multiplexName] = _process.inputStream + } + + // stderr + if(_processIO[StreamType.stderr]?.stream == null) + { + streams[StreamType.stderr.multiplexName] = _process.errorStream + } + + // exit value (as a stream) + InputStream exitValueInputStream = new InputGeneratorStream({ + try + { + future.get().toString() + } + catch(Throwable ignored) + { + // ok to ignore... will be part of the exit error stream... + } + }) + streams[StreamType.exitValue.multiplexName] = exitValueInputStream + + // exit error (as a stream) + InputStream exitErrorInputStream = new InputGeneratorStream({ + try + { + future.get().toString() + // no error + return null + } + catch(ExecutionException e) + { + GluGroovyJsonUtils.exceptionToJSON(e.cause) + } + catch(Throwable th) + { + GluGroovyJsonUtils.exceptionToJSON(th) + } + }) + streams[StreamType.exitError.multiplexName] = exitErrorInputStream + + def stream = new MultiplexedInputStream(streams) + stream.submitter = shell.submitter + return stream + } + + private def executeBlockingCall = { + + // if stdin then provide it to subprocess + _stdin?.withStream { InputStream sis -> + new BufferedOutputStream(_process.outputStream).withStream { os -> + os << new BufferedInputStream(sis) + } + } + + // make sure that the thread complete properly + [StreamType.stdout, StreamType.stderr].each { + _processIO[it]?.future?.get() + } + + // we wait for the process to be done + int exitValue = _process.waitFor() + + Map bytes = + GroovyCollectionsUtils.toMapKey([StreamType.stdout, StreamType.stderr]) { + _processIO[it]?.streamOfBytes?.toByteArray() ?: new byte[0] + } + + // handling failOnError flag + if(_failOnError && Execute.isFailure(exitValue)) + { + if(log.isDebugEnabled()) + { + log.debug("Error while executing command ${_commandLine}: ${exitValue}") + log.debug("output=${shell.toStringOutput(bytes[StreamType.stdout])}") + log.debug("error=${shell.toStringOutput(bytes[StreamType.stderr])}") + } + + ShellExecException exception = + new ShellExecException("Error while executing command ${_commandLine}: res=${exitValue} - output=${shell.toLimitedStringOutput(bytes[StreamType.stdout], 512)} - error=${shell.toLimitedStringOutput(bytes[StreamType.stderr], 512)}".toString()) + exception.res = exitValue + exception.output = bytes[StreamType.stdout] + exception.error = bytes[StreamType.stderr] + + throw exception + } + + // handling final output + switch(_requiredRes) + { + case 'stdout': + return shell.toStringOutput(bytes[StreamType.stdout]) + + case 'stdoutBytes': + return bytes[StreamType.stdout] + + case 'stderr': + return shell.toStringOutput(bytes[StreamType.stderr]) + + case 'stderrBytes': + return bytes[StreamType.stderr] + + case 'exitValue': + case "stdoutStream": + case "stderrStream": + case "exitValueStream": + case 'stream': + return exitValue + + case 'all': + return [ + exitValue: exitValue, + stdout: shell.toStringOutput(bytes[StreamType.stdout]), + stderr: shell.toStringOutput(bytes[StreamType.stderr]) + ] + + case 'allBytes': + return [ + exitValue: exitValue, + stdout: bytes[StreamType.stdout], + stderr: bytes[StreamType.stderr] + ] + + default: + throw new IllegalArgumentException("unknown [${args.res}] res value") + } + } + + private void initCommandLine() + { + _commandLine = shell.toStringCommandLine(args.command) + + if(_commandLine.startsWith('file:')) + { + _commandLine -= 'file:' + } + } + + private void initOutput(StreamType streamType, String argName = streamType.name()) + { + def output = args."${argName}" + + def map = [stream: NullOutputStream.INSTANCE] + + _processIO[streamType] = map + + if(output != null) + { + if(_requiredRes == "stream" || _requiredRes == "${argName}Stream") + throw new IllegalArgumentException("args.${argName}=[${output}] incompatible with arg.res=[${_requiredRes}]") + + map.stream = output + } + else + { + if(_requiredRes == "stream" || _requiredRes == "${argName}Stream") + map.stream = null + else + { + if(_requiredRes == argName || + _requiredRes == "${argName}Bytes" || + _requiredRes == 'all' || + _requiredRes == 'allBytes') + { + map.stream = new ByteArrayOutputStream() + map.streamOfBytes = map.stream + } + } + } + } + + private void startOutputThread(StreamType streamType, Closure inputStreamProvider) + { + def stream = _processIO[streamType]?.stream + + if(stream != null) + { + def consumeStream = { + inputStreamProvider().withStream { InputStream inputStream -> + // if it is a closure, invoke the closure + if(stream instanceof Closure) + { + stream(inputStream) + } + else + { + // read stdout + new BufferedInputStream(inputStream).withStream { + stream << it + } + } + } + } + + // need to consume output (in a separate thread!) + def future = new FutureTaskExecution(consumeStream) + future.description = "${_commandLine} > ${streamType.name()}" + _processIO[streamType].future = future + future.runAsync(shell.submitter) + } + } +} \ No newline at end of file diff --git a/utils/org.linkedin.glu.utils/src/main/groovy/org/linkedin/glu/groovy/utils/shell/ShellExecException.groovy b/utils/org.linkedin.glu.utils/src/main/groovy/org/linkedin/glu/groovy/utils/shell/ShellExecException.groovy new file mode 100644 index 00000000..f6391fda --- /dev/null +++ b/utils/org.linkedin.glu.utils/src/main/groovy/org/linkedin/glu/groovy/utils/shell/ShellExecException.groovy @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2011-2013 Yan Pujante + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package org.linkedin.glu.groovy.utils.shell + +/** + * @author yan@pongasoft.com */ +public class ShellExecException extends Exception +{ + private static final long serialVersionUID = 1L; + + int res + byte[] output + byte[] error + + ShellExecException() + { + } + + ShellExecException(s) + { + super(s) + } + + ShellExecException(s, throwable) + { + super(s, throwable) + } + + String getStringOutput() + { + return toStringOutput(output) + } + + String getStringError() + { + return toStringOutput(error) + } + + /** + * Converts the output into a string. Assumes that the encoding is UTF-8. Replaces all line feeds + * by '\n' and remove the last line feed. + */ + private String toStringOutput(byte[] output) + { + if(output == null) + return null + + return new String(output, "UTF-8").readLines().join('\n') + } +} \ No newline at end of file diff --git a/utils/org.linkedin.glu.utils/src/main/groovy/org/linkedin/glu/groovy/utils/shell/ShellImpl.groovy b/utils/org.linkedin.glu.utils/src/main/groovy/org/linkedin/glu/groovy/utils/shell/ShellImpl.groovy new file mode 100644 index 00000000..d1052bbb --- /dev/null +++ b/utils/org.linkedin.glu.utils/src/main/groovy/org/linkedin/glu/groovy/utils/shell/ShellImpl.groovy @@ -0,0 +1,1010 @@ +/* + * Copyright (c) 2010-2010 LinkedIn, Inc + * Portions Copyright (c) 2011-2013 Yan Pujante + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + + +package org.linkedin.glu.groovy.utils.shell + +import eu.medsea.mimeutil.MimeUtil +import org.apache.tools.ant.filters.ReplaceTokens +import org.apache.tools.ant.filters.ReplaceTokens.Token +import org.linkedin.glu.groovy.utils.GluGroovyLangUtils +import org.linkedin.glu.groovy.utils.collections.GluGroovyCollectionUtils +import org.linkedin.glu.groovy.utils.concurrent.FutureTaskExecutionThreadFactory +import org.linkedin.glu.groovy.utils.io.GluGroovyIOUtils +import org.linkedin.glu.utils.concurrent.OneThreadPerTaskSubmitter +import org.linkedin.glu.utils.concurrent.Submitter +import org.linkedin.groovy.util.ant.AntUtils +import org.linkedin.groovy.util.concurrent.GroovyConcurrentUtils +import org.linkedin.groovy.util.encryption.EncryptionUtils +import org.linkedin.groovy.util.io.GroovyIOUtils +import org.linkedin.groovy.util.io.fs.FileSystem +import org.linkedin.groovy.util.net.GroovyNetUtils +import org.linkedin.util.clock.Clock +import org.linkedin.util.clock.SystemClock +import org.linkedin.util.io.resource.Resource +import org.linkedin.util.lang.MemorySize +import org.linkedin.util.url.QueryBuilder + +import javax.management.Attribute +import javax.management.MBeanServerConnection +import javax.management.ObjectName +import javax.management.remote.JMXConnectorFactory +import javax.management.remote.JMXServiceURL +import java.util.concurrent.TimeoutException +import java.util.regex.Pattern +import org.slf4j.Logger +import org.slf4j.LoggerFactory + +/** + * contains the utility methods for the shell + * + * @author ypujante@linkedin.com + */ +def class ShellImpl implements Shell +{ + public static final String MODULE = ShellImpl.class.getName(); + public static final Logger log = LoggerFactory.getLogger(MODULE); + + static { + MimeUtil.registerMimeDetector("eu.medsea.mimeutil.detector.MagicMimeMimeDetector"); + } + + Clock clock = SystemClock.instance() + + // will delegate all the calls to fileSystem + @Delegate FileSystem fileSystem + + /** + * The charset to use for reading/writing files */ + def charset = 'UTF-8' + + /** + * Used when threads are needed + */ + protected Submitter _submitter + + synchronized Submitter getSubmitter() + { + if(_submitter == null) + _submitter = new OneThreadPerTaskSubmitter(new FutureTaskExecutionThreadFactory()) + return _submitter + } + + void setSubmitter(Submitter submitter) + { + _submitter = submitter + } + + Shell newShell(fileSystem) + { + return new ShellImpl(fileSystem: fileSystem, + charset: charset, + clock: clock, + submitter: _submitter) + } + + Collection getMimeTypes(file) + { + withInputStream(file) { InputStream is -> + MimeUtil.getMimeTypes(is).collect { it.toString() } + } as Collection + } + + Resource unzip(file) + { + return unzip(file, createTempDir()) + } + + Resource unzip(file, toDir) + { + file = toResource(file) + toDir = toResource(toDir) + ant { ant -> + ant.unzip(src: file.file, dest: toDir.file) + } + return toDir + } + + Resource untar(file) + { + return untar(file, createTempDir()) + } + + Resource untar(file, toDir) + { + file = toResource(file) + toDir = toResource(toDir) + def compression = 'none' + + def mimeTypes = getMimeTypes(file) + + if(mimeTypes.find { it == 'application/x-gzip'}) + { + compression = 'gzip' + } + + ant { ant -> + ant.untar(src: file.file, dest: toDir.file, compression: compression) + } + + return toDir + } + + /** + * Uncompresses the provided file + * @return the original file or the new one + */ + Resource gunzip(file) + { + file = toResource(file) + + def gunzipFilename + if(file.filename.endsWith('.gz')) + { + gunzipFilename = file.filename[0..-4] + } + else + { + if(file.filename.endsWith('.tgz')) + { + gunzipFilename = "${file.filename[0..-5]}.tar" + } + else + { + throw new IOException("unknown suffix ${file.filename}") + } + } + + def gunzipFile = gunzip(file, file.parentResource."${URLEncoder.encode(gunzipFilename, "UTF-8")}") + + if(file != gunzipFile) + { + rm(file) + } + + return gunzipFile + } + + /** + * Uncompresses the provided file (if compressed) into a new file + */ + Resource gunzip(file, toFile) + { + file = toResource(file) + toFile = toResource(toFile) + + if(!file.isDirectory() && getMimeTypes(file).find { it == 'application/x-gzip'}) + { + if(!toFile.exists()) + { + mkdirs(toFile.parentResource) + } + + ant { ant -> + ant.gunzip(src: file.file, dest: toFile.file) + } + + return toFile + } + else + { + return file + } + } + + /** + * Compresses the provided file (first make sure that the file is compressed) + * + * @return the gziped file or the original file if not zipped + */ + Resource gzip(file) + { + file = toResource(file) + + def gzipFile = gzip(file, file.parentResource."${URLEncoder.encode(file.filename, "UTF-8")}.gz") + + if(file != gzipFile) + { + rm(file) + } + + return gzipFile + } + + /** + * Compresses the provided file (first make sure that the file is compressed) + * + * @return the gziped file or the original file if not zipped + */ + Resource gzip(file, toFile) + { + file = toResource(file) + toFile = toResource(toFile) + + if(!file.isDirectory() && getMimeTypes(file).find { it != 'application/x-gzip'}) + { + ant { ant -> + ant.gzip(src: file.file, destfile: toFile.file) + } + + return toFile + } + else + { + return file + } + } + + /** + * Compresses each file in a folder + * + * @param fileOrFolder a file (behaves like {@link #gzip(Object)}) or a folder + * @param recurse if true then recursively process all folders + * @return a map with key is resource and value is delta in size (original - new) (note that it + * contains only the resources that are modified!) + */ + Map gzipFileOrFolder(fileOrFolder, boolean recurse) + { + def res = [:] + + fileOrFolder = toResource(fileOrFolder) + + if(fileOrFolder.isDirectory()) + { + fileOrFolder.ls().each { file -> + if(recurse) + { + res.putAll(gzipFileOrFolder(file, recurse)) + } + else + { + long size = file.size() + def gzipedFile = gzip(file) + if(gzipedFile != file) + res[gzipedFile] = size - gzipedFile.size() + } + } + } + else + { + long size = fileOrFolder.size() + def gzipedFile = gzip(fileOrFolder) + if(gzipedFile != fileOrFolder) + res[gzipedFile] = size - gzipedFile.size() + } + + return res + } + + Resource untarAndDecrypt(file, toDir, encryptionKeys) + { + + def tmpDir = createTempDir() + untar(file, tmpDir) + EncryptionUtils.decryptFiles(tmpDir.getFile(), toDir.getFile(), encryptionKeys) + rmdirs(tmpDir) + return toDir + } + + + /** + * Exporting ant access to the shell + */ + def ant(Closure closure) + { + return AntUtils.withBuilder { closure(it) } + } + + /** + * Fetches the file pointed to by the location. The location can be File, + * a String or URI and must contain a scheme. Example of locations: + * http://locahost:8080/file.txt', file:/tmp/file.txt, + * ivy:/org.linkedin/util-core/1.0.0. + */ + Resource fetch(location) + { + return fetch(location, null) + } + + /** + * Fetches the file pointed to by the location. The location can be File, + * a String or URI and must contain a scheme. Example of locations: + * http://locahost:8080/file.txt', file:/tmp/file.txt, + * ivy:/org.linkedin/util-core/1.0.0. The difference with the other fetch method + * is that it fetches the file in the provided destination rather than in the tmp space. + */ + Resource fetch(location, destination) + { + URI uri = GroovyNetUtils.toURI(location) + + if(uri == null) + return null + + Resource tempFile + if(destination) + { + tempFile = toResource(destination) + mkdirs(tempFile.parentResource) + } + else + { + tempFile = createTempDir() + } + + if(tempFile.isDirectory()) + { + def filename = GroovyNetUtils.guessFilename(uri) + tempFile = tempFile.createRelative(filename) + } + + GroovyIOUtils.fetchContent(location, tempFile.file) + + return tempFile + } + + /** + * Fetches the content of the location and returns it as a String or + * null if the location is not reachable + * + * @deprecated use {@link #cat(Object) instead + */ + String fetchContent(location) + { + return cat(location) + } + + /** + * Returns the content of the location as a String or + * null if the location is not reachable + */ + String cat(location) + { + try + { + return GroovyIOUtils.cat(location) + } + catch(Exception e) + { + if(log.isDebugEnabled()) + log.debug("[ignored] exception while catting content ${location}", e) + return null + } + } + + /** + * Issue a 'HEAD' request. The location should be an http or https link. + * + * @param location + * @return a map with the following entries: + * responseCode: 200, 404... {@link java.net.HttpURLConnection#getResponseCode()} + * responseMessage: message {@link java.net.HttpURLConnection#getResponseMessage()} + * headers: representing all the headers {@link java.net.URLConnection#getHeaderFields()} + */ + Map httpHead(location) + { + Map res = [:] + + URI uri = GroovyNetUtils.toURI(location) + + URL url = uri.toURL() + URLConnection cx = url.openConnection() + try + { + if(cx instanceof HttpURLConnection) + { + cx.requestMethod = 'HEAD' + cx.doInput = true + cx.doOutput = false + + cx.connect() + + res.responseCode = cx.responseCode + res.responseMessage = cx.responseMessage + res.headers = cx.headerFields + } + } + finally + { + if(cx.respondsTo('close')) + cx.close() + } + + return res + } + + /** + * Issue a 'POST' request. The location should be an http or https link. The request will be + * made with application/x-www-form-urlencoded content type. + * + * @param location + * @param parameters the parameters of the post as map of key value pairs (value can be a single + * value or a collection of values) + * @return a map with the following entries: + * responseCode: 200, 404... {@link java.net.HttpURLConnection#getResponseCode()} + * responseMessage: message {@link java.net.HttpURLConnection#getResponseMessage()} + * headers: representing all the headers {@link java.net.URLConnection#getHeaderFields()} + */ + Map httpPost(location, Map parameters) + { + Map res = [:] + + URI uri = GroovyNetUtils.toURI(location) + + URL url = uri.toURL() + URLConnection cx = url.openConnection() + try + { + if(cx instanceof HttpURLConnection) + { + cx.requestMethod = 'POST' + cx.setRequestProperty('Content-Type', 'application/x-www-form-urlencoded') + cx.doInput = true + cx.doOutput = true + + cx.connect() + + QueryBuilder qb = new QueryBuilder() + + parameters?.each { k, v -> + if(v != null) + { + if(v instanceof Collection) + qb.addParameters(k.toString(), v.collect { it.toString() } as String[]) + else + qb.addParameter(k.toString(), v.toString()) + } + } + + cx.outputStream << qb.toString() + cx.outputStream.close() + + res.responseCode = cx.responseCode + res.responseMessage = cx.responseMessage + res.headers = cx.headerFields + } + } + finally + { + if(cx.respondsTo('close')) + cx.close() + } + + return res + } + + /** + * Similarly to the unix grep command, checks the location one line at a time and returns + * all the lines which matches the pattern. + */ + Collection grep(location, pattern) + { + return (Collection) grep(location, pattern, null) + } + + /** + * Similarly to the unix grep command, checks the location one line at a time and returns + * all the lines which matches the pattern + * + * @param options options to the command: + * out: an object to output the lines which match (default to []) + * count: returns the count only (does not use out) + * maxCount: stop reading after maxCount matches + */ + def grep(location, pattern, options) + { + options = options ?: [:] + options = new HashMap(options) + def out = options.out ?: [] + int count = 0 + + def uri = GroovyNetUtils.toURI(location) + + if(pattern instanceof String) + { + pattern = Pattern.compile(pattern) + } + + try + { + def idx = 1 + GroovyIOUtils.eachLine(uri.toURL()) { line -> + if(pattern.matcher(line).find()) + { + count++ + if(!options.count) + out << line + if(count == options.maxCount) + return false + } + idx++ + return true + } + } + catch(Exception e) + { + if(log.isDebugEnabled()) + log.debug("[ignored] exception while grepping content ${location}", e) + } + + if(options.count) + { + return count + } + else + { + return out + } + } + + /** + * Executes a shell command... the command will be delegated straight to shell and the output of + * the shell command is returned. + */ + String exec(executable, executableArgs) + { + executable = chmodPlusX(executable) + doExec("${executable} ${toStringCommandLine(executableArgs)}".toString()) + } + + /** + * Executes a shell command... the command will be delegated straight to shell and the output of + * the shell command is returned. + */ + String exec(executable, Object... executableArgs) + { + exec(executable, executableArgs.collect { it }) + } + + /** + * Executes a shell command... the command will be delegated straight to shell and the output of + * the shell command is returned. + */ + String exec(String command) + { + return doExec(command) + } + + /** + * Executes a shell command... the command will be delegated straight to shell and the output of + * the shell command is returned. + */ + String exec(Collection command) + { + return doExec(command) + } + + /** + * @{inheritDoc} + */ + def exec(Map args) + { + if(args.pwd) + { + def pwd = toResource(args.pwd).file + args = GluGroovyCollectionUtils.xorMap(args, ['pwd']) + args.pwd = pwd + } + new ShellExec(shell: this, args: args).exec() + } + + /** + * @{inheritDoc} + */ + def demultiplexExecStream(InputStream execStream, + OutputStream stdout, + OutputStream stderr) + { + GluGroovyIOUtils.demultiplexExecStream(execStream, stdout, stderr) + } + + /** + * Shortcut/More efficient implementation of the more generic {@link #chmod(Object, Object) call + */ + Resource chmodPlusX(file) + { + file = toResource(file) + file.file.setExecutable(true) + return file + } + + /** + * Change the permission of the file + */ + Resource chmod(file, perm) + { + file = toResource(file) + exec(command: ['chmod', perm, file.file], res: 'exitValue') + return file + } + + Resource chmodRecursive(dir, perm) + { + eachChildRecurse(dir) { file -> + exec(command: ['chmod', perm, file.file], res: 'exitValue') + } + + return dir + } + + /** + * Invokes the closure with an MBeanServerConnection to the jmx control running + * on the vm started with the provided pid. The closure will be invoked with null + * if cannot determine the process. + */ + def withMBeanServerConnection(pid, Closure closure) + { + def serviceURL = extractJMXServiceURL(pid) + if(serviceURL) + { + def connector = null + try + { + connector = JMXConnectorFactory.connect(serviceURL) + } + catch(IOException e) + { + // ignored (connector remains null) + if(log.isDebugEnabled()) + { + log.debug("Ignored exception", e) + } + } + + try + { + use(MBeanServerConnectionCategory) { + closure(connector?.MBeanServerConnection) + } + } + finally + { + connector?.close() + } + } + else + { + closure(null) + } + } + + /** + * Extract the serviceURL using sun internal implementation + */ + private JMXServiceURL extractJMXServiceURL(pid) + { + if(pid == null) + return null + + String serviceURL = null + try + { + serviceURL = sun.management.ConnectorAddressLink.importFrom(pid as int) + } + catch (IOException ignored) + { + log.warn("Cannot find process ${pid}") + } + + if(serviceURL == null) + return null + else + return new JMXServiceURL(serviceURL) + } + + /** + * Waits for the condition to be true no longer than the timeout. true + * is returned if the condition was satisfied, false otherwise (if you specify + * noException) + * @param args.timeout how long max to wait + * @param args.heartbeat how long to wait between calling the condition + * @param args.noException to get false instead of an exception + */ + boolean waitFor(args, Closure condition) + { + try + { + GroovyConcurrentUtils.waitForCondition(clock, args.timeout, args.heartbeat, condition) + return true + } + catch (TimeoutException e) + { + if(args.noException) + return false + else + throw e + } + } + + /** + * Waits for the condition to be true no longer than the timeout. true + * is returned if the condition was satisfied, false otherwise + */ + boolean waitFor(Closure condition) + { + return waitFor([:], condition) + } + + /** + * Tail the location + * + * @params location the location of the file to tail + * @params maxLine the number of lines maximum to read + * + * @return the input stream to read the content of the tail + */ + InputStream tail(location, long maxLine) + { + return tail([location: location, maxLine: maxLine]) + } + + /** + * Tail the location + * + * @params args.location the location of the file to tail + * @params args.maxLine the number of lines maximum to read (-1 for the entire file) + * @params args.maxSize the maximum size to read (-1 for the entire file) + * + * @return the input stream to read the content of the tail + */ + InputStream tail(args) + { + File file = toResource(args.location)?.file + + if(file && file.exists()) + { + if(args.maxLine?.toString() == '-1' || args.maxSize?.toString() == '-1') + return new FileInputStream(file) + + def commandLine = ['tail'] + if(args.maxLine) + commandLine << "-${args.maxLine}" + if(args.maxSize) + commandLine << '-c' << MemorySize.parse(args.maxSize.toString()).sizeInBytes + commandLine << file.canonicalPath + return forkAndExec(commandLine) + } + else + return null + } + + /** + * Forks a process to execute the command line provided (as a single string) and returns the + * input stream of the process + */ + private InputStream forkAndExec(commandLine) + { + exec(command: commandLine, + redirectStderr: true, + res: 'stdoutStream') as InputStream + } + + /** + * Make sure that the command line is a string. + */ + protected String toStringCommandLine(commandLine) + { + if(commandLine instanceof GString) + commandLine = commandLine.toString() + + if(!(commandLine instanceof String)) + commandLine = commandLine.collect { it.toString() }.join(' ') + + return commandLine + } + + protected InputStream toInputStream(stream) + { + if(stream == null) + return stream + + if(stream instanceof InputStream) + return stream + + return new ByteArrayInputStream(stream.toString().getBytes(charset)) + } + + protected def doExec(commandLine) + { + return exec(command: commandLine, + failOnError: true, + res: 'all').stdout + } + + /** + * Converts the output into a string. Assumes that the encoding is UTF-8. Replaces all line feeds + * by '\n' and remove the last line feed. + */ + def toStringOutput(byte[] output) + { + return output ? new String(output, charset).readLines().join('\n') : "" + } + + /** + * Converts the output into a string with no more than maxChars characters. If there are more + * characters, then display '...' at the end. + */ + def toLimitedStringOutput(byte[] output, int maxChars) + { + def string = toStringOutput(output) + if(string?.size() > maxChars) + { + string = string.substring(0, maxChars) + "..." + } + return string + } + + /** + * @return true if there is a socket open on the server/port combination + */ + boolean listening(server, port) + { + ant { ant -> + ant.condition(property: 'listening') { + socket(server: server, port: port) + }.project.getProperty('listening') ? true : false + } as boolean + } + + /** + * Replaces the tokens provided in the map in the input. Token replacement is using ant token + * replacement class so in the input, the tokens are surrounded by the '@' sign. + * Example: + * + * 
+   * input = "abcd @myToken@"
+   * assert "abcd foo" == replaceTokens(input, [myToken: 'foo'])
+   *
+ */ + String replaceTokens(String input, Map tokens) + { + if(input == null) + return null + + ReplaceTokens rt = new ReplaceTokens(new StringReader(input)) + tokens?.each { k,v -> + rt.addConfiguredToken(new Token(key: k, value: v)) + } + + return rt.text + } + + /** + * Processes from through the replacement token mechanism and writes the result to + * to + * + * @param from anything that can be provided to {@link FileSystem#toResource(Object)} + * @param to anything that can be provided to {@link FileSystem#toResource(Object)} + * @param tokens a map of token + * @return to as a {@link Resource} + * @see #replaceTokens(String, Map) + */ + Resource replaceTokens(def from, def to, Map tokens) + { + to = toResource(to) + + withWriter(to) { Writer writer -> + withReader(from) { Reader reader -> + ReplaceTokens rt = new ReplaceTokens(reader) + tokens?.each { k,v -> + rt.addConfiguredToken(new Token(key: k, value: v)) + } + writer << rt + } + } + + return to + } + + /** + * Processes the content to the token replacement method. + * + * @see FileSystem#saveContent(Object, String) + */ + Resource saveContent(file, String content, Map tokens) + { + saveContent(file, replaceTokens(content, tokens)) + } + + /** + * Same as withInputStream but wraps in a reader using the {@link #charset} + * @param file anything that can be provided to {@link FileSystem#toResource(Object)} + * @return whatever the closure returns + */ + def withReader(file, Closure closure) + { + withInputStream(file) { InputStream is -> + is.withReader(charset, closure) + } + } + + /** + * Same as withOutputStream but wraps in a writer using the {@link #charset} + * @param file anything that can be provided to {@link FileSystem#toResource(Object)} + * @return whatever the closure returns + */ + def withWriter(file, Closure closure) + { + withOutputStream(file) { OutputStream os -> + os.withWriter(charset, closure) + } + } + + /** + * Runs the closure in a protected block that will not throw an exception but will return + * null in the case one happens + */ + def noException(Closure closure) + { + GluGroovyLangUtils.noException(closure) + } +} + +class MBeanServerConnectionCategory +{ + static def getAttribute(MBeanServerConnection self, objectName, String attribute) + { + objectName = toObjectName(objectName) + + return self.getAttribute(objectName, attribute) + } + + static Map getAttributes(MBeanServerConnection self, objectName, attributes) + { + objectName = toObjectName(objectName) + + attributes = attributes as String[] + + Map res = [:] + + self.getAttributes(objectName, attributes)?.each { Attribute attribute -> + res[attribute.name] = attribute.value + } + + return res + } + + static def invoke(MBeanServerConnection self, objectName, String methodName, parameters) + { + objectName = toObjectName(objectName) + + def info = self.getMBeanInfo(objectName) + + def operations = info.getOperations().findAll { + it.name == methodName && it.signature.size() == parameters.size() + } + + // if there is only one method with the given name the same number of parameters then we can call + // it + if(operations.size() != 1) + { + if(operations) + throw new UnsupportedOperationException("cannot use this form of invoke for overloaded methods") + else + throw new NoSuchMethodException(methodName) + } + + self.invoke(objectName, methodName, parameters as Object[], operations[0].signature.type as String[]) + } + + private static ObjectName toObjectName(objectName) + { + if(!(objectName instanceof ObjectName)) + objectName = new ObjectName(objectName.toString()) + + return objectName + } +} diff --git a/utils/org.linkedin.glu.utils/src/test/groovy/test/utils/shell/TestShell.groovy b/utils/org.linkedin.glu.utils/src/test/groovy/test/utils/shell/TestShell.groovy new file mode 100644 index 00000000..5e3dd9d1 --- /dev/null +++ b/utils/org.linkedin.glu.utils/src/test/groovy/test/utils/shell/TestShell.groovy @@ -0,0 +1,553 @@ +package test.utils.shell + +import com.sun.net.httpserver.Headers +import com.sun.net.httpserver.HttpExchange +import org.linkedin.glu.groovy.utils.shell.ShellExec +import org.linkedin.glu.groovy.utils.shell.ShellExecException +import org.linkedin.glu.groovy.utils.shell.ShellImpl +import org.linkedin.groovy.util.collections.GroovyCollectionsUtils +import org.linkedin.groovy.util.io.GroovyIOUtils +import org.linkedin.groovy.util.io.fs.FileSystemImpl +import org.linkedin.groovy.util.ivy.IvyURLHandler +import org.linkedin.groovy.util.net.GroovyNetUtils +import org.linkedin.groovy.util.net.SingletonURLStreamHandlerFactory +import org.linkedin.util.io.resource.Resource + +/** + * @author yan@pongasoft.com */ +public class TestShell extends GroovyTestCase +{ + void testFetch() + { + FileSystemImpl.createTempFileSystem() { org.linkedin.groovy.util.io.fs.FileSystem fs -> + def shell = new ShellImpl(fileSystem: fs) + + Resource tempFile = fs.tempFile() + assertFalse(tempFile.exists()) + fs.saveContent(tempFile, "this is a test") + assertTrue(tempFile.exists()) + + // on the other end if we provide a uri it will 'fetch' it to a different location + Resource fetchedFile = shell.fetch(tempFile) + assertNotSame(tempFile.file.canonicalPath, fetchedFile.file.canonicalPath) + + // we make sure that the file was copied entirely + assertEquals("this is a test", fs.readContent(fetchedFile)) + + // we now remove the temp file and we make sure that fetch throws an exception + fs.rm(tempFile) + assertFalse(tempFile.exists()) + shouldFail(FileNotFoundException) { shell.fetch(tempFile.toURI()) } + } + } + + /** + * Using shell.fetch on a remote url (http) + */ + void testFetchRemote() + { + FileSystemImpl.createTempFileSystem() { org.linkedin.groovy.util.io.fs.FileSystem fs -> + def shell = new ShellImpl(fileSystem: fs) + + String response + Headers requestHeaders + + def handler = { HttpExchange t -> + requestHeaders = t.requestHeaders + t.sendResponseHeaders(200, response.length()); + OutputStream os = t.getResponseBody(); + os.write(response.getBytes()); + os.close(); + } + + GroovyNetUtils.withHttpServer(0, ['/content': handler]) { int port -> + + File root = fs.root.file + + File tmpFile = new File(root, 'foo.txt') + + response = "abc" + + shell.fetch("http://localhost:${port}/content", tmpFile) + assertEquals("abc", tmpFile.text) + // no authorization header should be present! + assertFalse(requestHeaders.containsKey('Authorization')) + + response = "def" + shell.fetch("http://u1:p1@localhost:${port}/content", tmpFile) + assertEquals("def", tmpFile.text) + // authorization header should be present and properly base64ified + assertEquals("Basic ${'u1:p1'.bytes.encodeBase64()}", + requestHeaders['Authorization'].iterator().next()) + } + } + } + + /** + * Use a local repo to fetch a file which content is well known. + */ + void testFetchWithIvy() + { + FileSystemImpl.createTempFileSystem() { org.linkedin.groovy.util.io.fs.FileSystem fs -> + + def ivySettings = new File("./src/test/resources/ivysettings.xml").canonicalFile.toURI() + + def factory = new SingletonURLStreamHandlerFactory() + factory.registerHandler('ivy') { + return new IvyURLHandler(ivySettings) + } + URL.setURLStreamHandlerFactory(factory) + + def shell = new ShellImpl(fileSystem: fs) + + def file = shell.fetch('ivy:/test.agent.impl/myArtifact/1.0.0') + + assertEquals(new File("./src/test/resources/test/agent/impl/myArtifact/1.0.0/myArtifact-1.0.0.jar").getText(), + fs.readContent(file)) + + assertEquals('myArtifact-1.0.0.jar', GroovyNetUtils.guessFilename(new URI('ivy:/test.agent.impl/myArtifact/1.0.0'))) + + file = shell.fetch('ivy:/test.agent.impl/myArtifact/1.0.0/text') + + assertEquals(new File("./src/test/resources/test/agent/impl/myArtifact/1.0.0/myArtifact-1.0.0.txt").getText(), + fs.readContent(file)) + } + } + + /** + * Test that we can exec commands. + */ + void testExec() + { + FileSystemImpl.createTempFileSystem() { org.linkedin.groovy.util.io.fs.FileSystem fs -> + def shell = new ShellImpl(fileSystem: fs) + + // non existent script + try + { + shell.exec("/non/existent/666") + fail("should fail") + } + catch(ShellExecException e) + { + assertEquals(127, e.res) + assertEquals('', e.stringOutput) + String shellScriptError = 'bash: /non/existent/666: No such file or directory' + assertEquals(shellScriptError, e.stringError) + assertEquals('Error while executing command /non/existent/666: res=127 - output= - error=' + + shellScriptError, + e.message) + } + + def shellScript = shell.fetch("./src/test/resources/shellScriptTestCapabilities.sh") + + // we make sure that the script is not executable because exec changes the exec flag + // automatically + fs.chmod(shellScript, '-x') + + // we make sure that the other syntax works + assertEquals("Hello", shell.exec(shellScript)) + assertEquals("Hello a b c", shell.exec(shellScript, "a b c")) + assertEquals("Hello a b c", shell.exec(shellScript, ["a", "b", "c"])) + assertEquals("Hello a b c", shell.exec(shellScript, "a", "b", "c")) + assertEquals("Hello a b c", shell.exec(shellScript.file, ["a", "b", "c"])) + + // and we can call with resource as well + assertEquals("Hello a b c", shell.exec("${shellScript} a b c")) + + // now we try with a file directly: + shellScript = shellScript.file.canonicalPath + assertEquals("Hello a b c", shell.exec("${shellScript} a b c")) + assertEquals(" 1 1 6".trim(), shell.exec("${shellScript} | wc").trim()) + + // we make the shell script non executable + fs.chmod(shellScript, '-x') + try + { + shell.exec("${shellScript} a b c") + fail("should fail") + } + catch(ShellExecException e) + { + assertEquals(126, e.res) + assertEquals('', e.stringOutput) + String shellScriptError = + "bash: ${shellScript}: Permission denied".toString() + assertEquals(shellScriptError, e.stringError) + assertEquals("Error while executing command ${shellScript} a b c: " + + "res=126 - output= - error=${shellScriptError}", + e.message) + } + } + } + + void testGenericExec() + { + FileSystemImpl.createTempFileSystem() { org.linkedin.groovy.util.io.fs.FileSystem fs -> + def shell = new ShellImpl(fileSystem: fs) + def shellScript = shell.fetch("./src/test/resources/shellScriptTestShellExec.sh") + // let's make sure it is executable + fs.chmod(shellScript, '+x') + + // stdout only + checkShellExec(shell, [command: [shellScript, "-1"]], 0, "this goes to stdout\n", "") + + // both stdout and stderr in their proper channel + checkShellExec(shell, [command: [shellScript, "-1", "-2"]], 0, "this goes to stdout\n", "this goes to stderr\n") + + // redirecting stderr to stdout + checkShellExec(shell, [command: [shellScript, "-1", "-2"], redirectStderr: true], 0, "this goes to stdout\nthis goes to stderr\n", "") + + // changing stdout + def myStdout = new ByteArrayOutputStream() + checkShellExec(shell, [command: [shellScript, "-1", "-2"], stdout: myStdout], 0, "", "this goes to stderr\n") { + // implementation note: output here is not "processed" (see javadoc) so need to add the + // final \n character + assertEquals("this goes to stdout\n", new String(myStdout.toByteArray(), "UTF-8")) + myStdout.reset() + } + + // changing stderr + def myStderr = new ByteArrayOutputStream() + checkShellExec(shell, [command: [shellScript, "-1", "-2"], stderr: myStderr], 0, "this goes to stdout\n", "") { + // implementation note: output here is not "processed" (see javadoc) so need to add the + // final \n character + assertEquals("this goes to stderr\n", new String(myStderr.toByteArray(), "UTF-8")) + myStderr.reset() + } + + // testing for failure/exit value + checkShellExec(shell, [command: [shellScript, "-1", "-e"], failOnError: false], 1, "this goes to stdout\n", "") + + // test that when there is a failure, then an exception is properly generated if failOnError + // is not defined + def errorMsg = shouldFail(ShellExecException) { + shell.exec(command: [shellScript, "-1", "-e"]) + } + assertTrue(errorMsg.endsWith("res=1 - output=this goes to stdout - error=")) + + // test that when there is a failure, then an exception is properly generated if failOnError + // is set to true + errorMsg = shouldFail(ShellExecException) { + shell.exec(command: [shellScript, "-1", "-e"], failOnError: true) + } + assertTrue(errorMsg.endsWith("res=1 - output=this goes to stdout - error=")) + + // reading from stdin + checkShellExec(shell, [command: [shellScript, "-1", "-c"], stdin: "abc\ndef\n"], 0, "this goes to stdout\nabc\ndef\n", "") + + // testing for stdoutStream + InputStream stdout = shell.exec(command: [shellScript, "-1", "-2"], res: "stdoutStream") + assertEquals("this goes to stdout\n", stdout.text) + + // testing for stdoutStream + InputStream stderr = shell.exec(command: [shellScript, "-1", "-2"], res: "stderrStream") + assertEquals("this goes to stderr\n", stderr.text) + + // testing for stream + InputStream stream = shell.exec(command: [shellScript, "-1", "-2", "-e"], failOnError: false, res: "stream") + + myStdout = new ByteArrayOutputStream() + myStderr = new ByteArrayOutputStream() + + assertEquals(1, shell.demultiplexExecStream(stream, myStdout, myStderr)) + assertEquals("this goes to stdout\n", new String(myStdout.toByteArray(), "UTF-8")) + assertEquals("this goes to stderr\n", new String(myStderr.toByteArray(), "UTF-8")) + + stream = shell.exec(command: [shellScript, "-1", "-2", "-e"], failOnError: true, res: "stream") + + myStdout = new ByteArrayOutputStream() + myStderr = new ByteArrayOutputStream() + + errorMsg = shouldFail(ShellExecException) { + shell.demultiplexExecStream(stream, myStdout, myStderr) + } + assertTrue(errorMsg.endsWith("res=1 - output= - error=")) + assertEquals("this goes to stdout\n", new String(myStdout.toByteArray(), "UTF-8")) + assertEquals("this goes to stderr\n", new String(myStderr.toByteArray(), "UTF-8")) + + // testing pwd + checkShellExec(shell, [command: ["pwd"]], 0, "${new File(".").canonicalPath}\n", "") + def pwdDir = shell.mkdirs("/pwd") + checkShellExec(shell, [command: ["pwd"], pwd: "/pwd"], 0, "${pwdDir.file.canonicalPath}\n", "") + + ProcessBuilder pb = new ProcessBuilder(ShellExec.buildCommandLine('pwd')) + pb.directory(shell.toResource("/pwdDoNotExist").file) + String errorMessage = null + try + { + pb.start() + } + catch(IOException e) + { + errorMessage = e.message + } + + checkShellExec(shell, [command: ["pwd"], pwd: "/pwdDoNotExist", failOnError: false], 2, "", "${errorMessage}") + + // testing env + checkShellExec(shell, [command: ['echo $HOME']], 0, "${System.getenv().HOME}\n", "") + + // changing environment variable + def homeDir = shell.mkdirs("/home") + checkShellExec(shell, [command: ['echo $HOME'], env: [HOME: homeDir.file.canonicalPath]], 0, "${homeDir.file.canonicalPath}\n", "") + + // removing environment variable + checkShellExec(shell, [command: ['echo $HOME'], env: [HOME: null]], 0, "\n", "") + + // not inheriting + checkShellExec(shell, [command: ['echo $HOME'], inheritEnv: false], 0, "\n", "") + } + } + + private static void checkShellExec(shell, commands, exitValue, stdout, stderr) + { + checkShellExec(shell, commands, exitValue, stdout, stderr, null) + } + + private static void checkShellExec(shell, commands, exitValue, stdout, stderr, Closure cl) + { + assertEquals(stdout.trim(), shell.exec(*:commands)) + if(cl) cl() + assertEquals(exitValue, shell.exec(*:commands, res: "exitValue")) + if(cl) cl() + assertEquals(exitValue.toString(), shell.exec(*:commands, res: "exitValueStream").text) + if(cl) cl() + assertEquals(stdout.trim(), shell.exec(*:commands, res: "stdout")) + if(cl) cl() + assertEquals(stdout.getBytes("UTF-8"), shell.exec(*:commands, res: "stdoutBytes")) + if(cl) cl() + assertEquals(stderr.trim(), shell.exec(*:commands, res: "stderr")) + if(cl) cl() + assertEquals(stderr.getBytes("UTF-8"), shell.exec(*:commands, res: "stderrBytes")) + if(cl) cl() + assertEquals([exitValue: exitValue, stdout: stdout.trim(), stderr: stderr.trim()], + shell.exec(*:commands, res: "all")) + if(cl) cl() + def res = shell.exec(*:commands, res: "allBytes") + assertEquals(exitValue, res.exitValue) + assertEquals(stdout.getBytes("UTF8"), res.stdout) + assertEquals(stderr.getBytes("UTF8"), res.stderr) + if(cl) cl() + } + + void testTail() + { + FileSystemImpl.createTempFileSystem() { org.linkedin.groovy.util.io.fs.FileSystem fs -> + def shell = new ShellImpl(fileSystem: fs) + + def content = new StringBuilder() + + def f = fs.withOutputStream('testFile') { file, out -> + (1..1000).each { lineNumber -> + def line = "this is line ${lineNumber}\n" + out.write(line.getBytes('UTF-8')) + content = content << line.toString() + } + return file + } + + assertEquals("""this is line 997 +this is line 998 +this is line 999 +this is line 1000 +""", shell.tail(f, 4).text) + + assertEquals(content.toString(), shell.tail(f, -1).text) + + assertNull(shell.tail(location: 'do not exists')) + } + } + + /** + * Test that untar handles gzip properly + */ + void testUntar() + { + FileSystemImpl.createTempFileSystem() { org.linkedin.groovy.util.io.fs.FileSystem fs -> + def shell = new ShellImpl(fileSystem: fs) + + ["./src/test/resources/testUntar_tar", "./src/test/resources/testUntar_tgz"].each { file -> + def fetchedFile = new File(file).canonicalFile.toURI() + + def untarred = shell.untar(shell.fetch(fetchedFile)) + + assertEquals("for ${file}", ['a.txt', 'b.txt', 'c.txt'], shell.ls(untarred).filename.sort()) + } + } + } + + /** + * Test the grep capability + */ + void testGrep() + { + FileSystemImpl.createTempFileSystem() { org.linkedin.groovy.util.io.fs.FileSystem fs -> + def shell = new ShellImpl(fileSystem: fs) + + Resource tempFile = fs.tempFile() + fs.saveContent(tempFile, """line 1 abc +line 2 def +line 3 abcdef +""") + + assertEquals(['line 1 abc', 'line 3 abcdef'], shell.grep(tempFile, /abc/)) + assertEquals(2, shell.grep(tempFile, /abc/, [count: true])) + assertEquals(['line 1 abc'], shell.grep(tempFile, /abc/, [maxCount: 1])) + assertEquals(1, shell.grep(tempFile, /abc/, [count: true, maxCount: 1])) + + // test for 'out' option + def sw = new StringWriter() + assertEquals(sw, shell.grep(tempFile, /abc/, [out: sw])) + assertEquals('line 1 abcline 3 abcdef', sw.toString()) + } + } + + + /** + * Test for listening capability + */ + void testListening() + { + def shell = new ShellImpl() + assertFalse(shell.listening('localhost', 60000)) + + def serverSocket = new ServerSocket(60000) + + def thread = Thread.startDaemon { + try + { + while(true) + { + serverSocket.accept { socket -> + socket.close() + } + } + } + catch (InterruptedIOException e) + { + if(log.isDebugEnabled()) + log.debug("ok because the thread is interrupted... ignored", e) + } + } + + assertTrue(shell.listening('localhost', 60000)) + thread.interrupt() + thread.join(1000) + } + + void testGzip() + { + FileSystemImpl.createTempFileSystem() { org.linkedin.groovy.util.io.fs.FileSystem fs -> + def shell = new ShellImpl(fileSystem: fs) + + def root = shell.toResource('/root') + + shell.saveContent('/root/dir1/a.txt', 'this is a test aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') + shell.saveContent('/root/dir1/b.txt', 'this is a test baaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') + shell.saveContent('/root/dir1/dir2/c.txt', 'this is a test caaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') + shell.saveContent('/root/dir1/dir2/d.txt', 'this is a test daaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa') + shell.saveContent('/root/dir1/dir2/e.txt', 'e') // this will generate a negative delta! + + def expectedResult = leavesPaths(root) + def originalSizes = GroovyCollectionsUtils.toMapKey(expectedResult) { shell.toResource(it).size() } + + assertEquals('/root/dir1/dir2/d.txt.gz', shell.gzip('/root/dir1/dir2/d.txt').path) + + expectedResult << '/root/dir1/dir2/d.txt.gz' + expectedResult.remove('/root/dir1/dir2/d.txt') + + assertTrue(GroovyCollectionsUtils.compareIgnoreType(expectedResult, leavesPaths(root))) + assertEquals('this is a test daaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', shell.gunzip('/root/dir1/dir2/d.txt.gz', '/gunzip/d.txt').file.text) + + assertEquals('/root/dir1/dir2/d.txt', shell.gunzip('/root/dir1/dir2/d.txt.gz').path) + expectedResult << '/root/dir1/dir2/d.txt' + expectedResult.remove('/root/dir1/dir2/d.txt.gz') + assertTrue(GroovyCollectionsUtils.compareIgnoreType(expectedResult, leavesPaths(root))) + + assertEquals('this is a test daaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', shell.readContent('/root/dir1/dir2/d.txt')) + + // not recursive => no changes + assertEquals([:], shell.gzipFileOrFolder('/root', false)) + assertTrue(GroovyCollectionsUtils.compareIgnoreType(expectedResult, leavesPaths(root))) + + def res = shell.gzipFileOrFolder('/root', true) + expectedResult = ['/root/dir1/a.txt.gz', '/root/dir1/b.txt.gz', '/root/dir1/dir2/c.txt.gz', '/root/dir1/dir2/d.txt.gz', '/root/dir1/dir2/e.txt.gz'] + def sizes = GroovyCollectionsUtils.toMapKey(expectedResult) { shell.toResource(it).size() } + + sizes.each { path, size -> + assertEquals(originalSizes[path - '.gz'] - size, res[shell.toResource(path)]) + } + + assertEquals(expectedResult, res.keySet().path.sort()) + assertEquals(expectedResult, leavesPaths(root).toArray().sort()) + } + } + + void testRecurse() + { + FileSystemImpl.createTempFileSystem() { org.linkedin.groovy.util.io.fs.FileSystem fs -> + def shell = new ShellImpl(fileSystem: fs) + + def root = shell.toResource('/root') + shell.saveContent('/root/dir1/a.txt', 'this is a test a') + shell.saveContent('/root/dir1/b.txt', 'this is a test b') + shell.saveContent('/root/dir1/dir2/c.txt', 'this is a test c') + shell.saveContent('/root/dir1/dir2/d.txt', 'this is a test d') + shell.saveContent('/root/e.txt', 'e') + + // every file and dir under root + def files = [] + fs.eachChildRecurse(root) { r -> + files << r.path + } + files.sort() + + def expectedFiles = ['/root/dir1', '/root/e.txt', '/root/dir1/a.txt', '/root/dir1/b.txt', '/root/dir1/dir2', '/root/dir1/dir2/c.txt', '/root/dir1/dir2/d.txt'].sort() + assertEquals(expectedFiles, files) + + // find only dirs under root + def dirs = fs.findAll('/root') { r -> + if (r.isDirectory()) { + return true + } else { + return false + } + } + + def expectedDirs = ['/root/dir1', '/root/dir1/dir2'] + assertEquals(expectedDirs, dirs.collect {it.path}) + + // make everything non-writable + shell.chmodRecursive(root, "u-w") + def writableFiles = files.findAll { shell.toResource(it).file.canWrite() } + assertEquals([], writableFiles) + + // make it writable now + shell.chmodRecursive(root, "u+w") + writableFiles = files.findAll { shell.toResource(it).file.canWrite() } + assertEquals(files, writableFiles) + } + } + + void testReplaceTokens() + { + FileSystemImpl.createTempFileSystem() { org.linkedin.groovy.util.io.fs.FileSystem fs -> + def shell = new ShellImpl(fileSystem: fs) + + assertEquals('abc foo efg bar hij foo', + shell.replaceTokens('abc @token1@ efg @token2@ hij @token1@', + [token1: 'foo', token2: 'bar'])) + + Resource testFile = shell.saveContent('test.txt', 'abc @token1@ efg @token2@ hij @token1@', + [token1: 'foo', token2: 'bar']) + + assertEquals('abc foo efg bar hij foo', shell.cat(testFile)) + } + } + + private def leavesPaths(Resource root) + { + new TreeSet(GroovyIOUtils.findAll(root) { !it.isDirectory() }.collect { it.path }) + } +} \ No newline at end of file diff --git a/agent/org.linkedin.glu.agent-impl/src/test/resources/ivysettings.xml b/utils/org.linkedin.glu.utils/src/test/resources/ivysettings.xml similarity index 100% rename from agent/org.linkedin.glu.agent-impl/src/test/resources/ivysettings.xml rename to utils/org.linkedin.glu.utils/src/test/resources/ivysettings.xml diff --git a/agent/org.linkedin.glu.agent-impl/src/test/resources/shellScriptTestCapabilities.sh b/utils/org.linkedin.glu.utils/src/test/resources/shellScriptTestCapabilities.sh similarity index 100% rename from agent/org.linkedin.glu.agent-impl/src/test/resources/shellScriptTestCapabilities.sh rename to utils/org.linkedin.glu.utils/src/test/resources/shellScriptTestCapabilities.sh diff --git a/utils/org.linkedin.glu.utils/src/test/resources/shellScriptTestShellExec.sh b/utils/org.linkedin.glu.utils/src/test/resources/shellScriptTestShellExec.sh new file mode 100755 index 00000000..f0ad2a3d --- /dev/null +++ b/utils/org.linkedin.glu.utils/src/test/resources/shellScriptTestShellExec.sh @@ -0,0 +1,18 @@ +#! /bin/bash + +# get script options +while getopts "12ec" opt ; do + case $opt in + 1 ) echo "this goes to stdout" + ;; + 2 ) echo "this goes to stderr" >&2 + ;; + e ) exit 1 + ;; + # this wait for stdin + c ) cat + ;; + esac +done + +exit 0 diff --git a/agent/org.linkedin.glu.agent-impl/src/test/resources/test/agent/impl/myArtifact/1.0.0/myArtifact-1.0.0.ivy b/utils/org.linkedin.glu.utils/src/test/resources/test/agent/impl/myArtifact/1.0.0/myArtifact-1.0.0.ivy similarity index 100% rename from agent/org.linkedin.glu.agent-impl/src/test/resources/test/agent/impl/myArtifact/1.0.0/myArtifact-1.0.0.ivy rename to utils/org.linkedin.glu.utils/src/test/resources/test/agent/impl/myArtifact/1.0.0/myArtifact-1.0.0.ivy diff --git a/agent/org.linkedin.glu.agent-impl/src/test/resources/test/agent/impl/myArtifact/1.0.0/myArtifact-1.0.0.jar b/utils/org.linkedin.glu.utils/src/test/resources/test/agent/impl/myArtifact/1.0.0/myArtifact-1.0.0.jar similarity index 100% rename from agent/org.linkedin.glu.agent-impl/src/test/resources/test/agent/impl/myArtifact/1.0.0/myArtifact-1.0.0.jar rename to utils/org.linkedin.glu.utils/src/test/resources/test/agent/impl/myArtifact/1.0.0/myArtifact-1.0.0.jar diff --git a/agent/org.linkedin.glu.agent-impl/src/test/resources/test/agent/impl/myArtifact/1.0.0/myArtifact-1.0.0.txt b/utils/org.linkedin.glu.utils/src/test/resources/test/agent/impl/myArtifact/1.0.0/myArtifact-1.0.0.txt similarity index 100% rename from agent/org.linkedin.glu.agent-impl/src/test/resources/test/agent/impl/myArtifact/1.0.0/myArtifact-1.0.0.txt rename to utils/org.linkedin.glu.utils/src/test/resources/test/agent/impl/myArtifact/1.0.0/myArtifact-1.0.0.txt diff --git a/agent/org.linkedin.glu.agent-impl/src/test/resources/testUntar_tar b/utils/org.linkedin.glu.utils/src/test/resources/testUntar_tar similarity index 100% rename from agent/org.linkedin.glu.agent-impl/src/test/resources/testUntar_tar rename to utils/org.linkedin.glu.utils/src/test/resources/testUntar_tar diff --git a/agent/org.linkedin.glu.agent-impl/src/test/resources/testUntar_tgz b/utils/org.linkedin.glu.utils/src/test/resources/testUntar_tgz similarity index 100% rename from agent/org.linkedin.glu.agent-impl/src/test/resources/testUntar_tgz rename to utils/org.linkedin.glu.utils/src/test/resources/testUntar_tgz