From f437027b7440c57114f2df6db11a175dbdb6a47f Mon Sep 17 00:00:00 2001 From: Yan Pujante Date: Wed, 27 Mar 2013 09:03:18 -1000 Subject: [PATCH] #135: workaround memory leak issue in groovy 2.0.7 --- .../agent/impl/capabilities/AntHelper.groovy | 51 ------------------- .../agent/impl/capabilities/ShellImpl.groovy | 5 +- .../command/CommandGluScriptFactory.groovy | 8 ++- .../script/FromClassNameScriptFactory.groovy | 8 +++ .../script/FromLocationScriptFactory.groovy | 20 +++++++- .../agent/impl/script/ScriptDefinition.groovy | 34 ++++++------- .../agent/impl/script/ScriptFactory.groovy | 6 +++ .../impl/script/ScriptManagerImpl.groovy | 10 +++- .../impl/zookeeper/ZooKeeperStorage.groovy | 7 ++- .../test/agent/perf/TestPerfTimers.groovy | 3 +- build.gradle | 9 +++- project-spec.groovy | 4 +- .../core/model/SystemFilterBuilder.groovy | 22 +++++--- 13 files changed, 99 insertions(+), 88 deletions(-) delete mode 100644 agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/capabilities/AntHelper.groovy diff --git a/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/capabilities/AntHelper.groovy b/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/capabilities/AntHelper.groovy deleted file mode 100644 index 76b76076..00000000 --- a/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/capabilities/AntHelper.groovy +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright (c) 2010-2010 LinkedIn, Inc - * - * Licensed under the Apache License, Version 2.0 (the "License"); you may not - * use this file except in compliance with the License. You may obtain a copy of - * the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations under - * the License. - */ - - -package org.linkedin.glu.agent.impl.capabilities - -import org.apache.tools.ant.Project -import org.apache.tools.ant.BuildException - -/** - * Helper methods for ant - * - * @author ypujante@linkedin.com - */ -class AntHelper -{ - /** - * Executes the closure with a builder and make sure to catch BuildException - * to propertly unwrap them - */ - static def withBuilder(Closure closure) - { - AntBuilder builder = new AntBuilder() - // removes info messages... - builder.project.buildListeners[0].messageOutputLevel = Project.MSG_WARN - try - { - return closure(builder) - } - catch (BuildException e) - { - if(e.cause) - throw e.cause - else - throw e - } - } -} diff --git a/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/capabilities/ShellImpl.groovy b/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/capabilities/ShellImpl.groovy index 906a8c41..ee752078 100644 --- a/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/capabilities/ShellImpl.groovy +++ b/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/capabilities/ShellImpl.groovy @@ -1,6 +1,6 @@ /* * Copyright (c) 2010-2010 LinkedIn, Inc - * Portions Copyright (c) 2011 Yan Pujante + * Portions Copyright (c) 2011-2013 Yan Pujante * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -20,6 +20,7 @@ package org.linkedin.glu.agent.impl.capabilities import eu.medsea.mimeutil.MimeUtil import org.linkedin.glu.groovy.utils.collections.GluGroovyCollectionUtils +import org.linkedin.groovy.util.ant.AntUtils import java.util.concurrent.TimeoutException import java.util.regex.Pattern @@ -319,7 +320,7 @@ def class ShellImpl implements Shell */ def ant(Closure closure) { - return AntHelper.withBuilder { closure(it) } + return AntUtils.withBuilder { closure(it) } } /** diff --git a/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/command/CommandGluScriptFactory.groovy b/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/command/CommandGluScriptFactory.groovy index 6f8448f2..12c95b4d 100644 --- a/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/command/CommandGluScriptFactory.groovy +++ b/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/command/CommandGluScriptFactory.groovy @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012 Yan Pujante + * Copyright (c) 2012-2013 Yan Pujante * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -34,6 +34,12 @@ public class CommandGluScriptFactory implements ScriptFactory new CommandGluScript(ioStorage: ioStorage) } + @Override + void destroyScript(ScriptConfig scriptConfig) + { + // nothing to do here + } + @Override def toExternalRepresentation() { diff --git a/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/script/FromClassNameScriptFactory.groovy b/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/script/FromClassNameScriptFactory.groovy index 33e5be5a..b3c9be98 100644 --- a/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/script/FromClassNameScriptFactory.groovy +++ b/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/script/FromClassNameScriptFactory.groovy @@ -1,5 +1,6 @@ /* * Copyright (c) 2010-2010 LinkedIn, Inc + * Portions Copyright (c) 2013 Yan Pujante * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -72,6 +73,13 @@ def class FromClassNameScriptFactory implements ScriptFactory, Serializable return _script } + @Override + void destroyScript(ScriptConfig scriptConfig) + { + _jarFiles?.each { scriptConfig.shell.rm(it) } + _classLoader = null + } + String toString() { if(_classPath) diff --git a/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/script/FromLocationScriptFactory.groovy b/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/script/FromLocationScriptFactory.groovy index ef1a4a57..86168e4c 100644 --- a/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/script/FromLocationScriptFactory.groovy +++ b/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/script/FromLocationScriptFactory.groovy @@ -1,6 +1,6 @@ /* * Copyright (c) 2010-2010 LinkedIn, Inc - * Portions Copyright (c) 2011 Yan Pujante + * Portions Copyright (c) 2011-2013 Yan Pujante * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -31,6 +31,7 @@ def class FromLocationScriptFactory implements ScriptFactory, Serializable private final def _location private def _scriptFile private transient def _script + private transient GroovyClassLoader _gcl FromLocationScriptFactory(location) { @@ -44,13 +45,28 @@ def class FromLocationScriptFactory implements ScriptFactory, Serializable if(!_scriptFile?.exists()) _scriptFile = scriptConfig.shell.fetch(_location) - Class scriptClass = new GroovyClassLoader(getClass().classLoader).parseClass(_scriptFile.file) + _gcl = new GroovyClassLoader(getClass().classLoader) + + Class scriptClass = _gcl.parseClass(_scriptFile.file) + _script = scriptClass.newInstance() } return _script } + @Override + void destroyScript(ScriptConfig scriptConfig) + { + if(_scriptFile?.exists()) + scriptConfig.shell.rm(_scriptFile) + + _scriptFile = null + _script = null + _gcl.clearCache() + _gcl = null + } + String toString() { return "FromLocationScriptFactory[${_location}]".toString(); diff --git a/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/script/ScriptDefinition.groovy b/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/script/ScriptDefinition.groovy index 542eea47..9ab2eae9 100644 --- a/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/script/ScriptDefinition.groovy +++ b/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/script/ScriptDefinition.groovy @@ -1,5 +1,6 @@ /* * Copyright (c) 2010-2010 LinkedIn, Inc + * Portions Copyright (c) 2013 Yan Pujante * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -31,52 +32,51 @@ class ScriptDefinition implements Serializable, Externable final MountPoint mountPoint final MountPoint parent - final ScriptFactory scriptFactory + final ScriptFactory scriptFactory = null // set to null always so that it does not get serialized final def initParameters final def _scriptFactoryArgs + final transient ScriptFactory _scriptFactory // not serialized ScriptDefinition(MountPoint mountPoint, MountPoint parent, ScriptFactory scriptFactory, initParameters) { - // parent is required unless mountpoint is root... + // parent is required unless mountPoint is root... assert parent != null || mountPoint == MountPoint.ROOT this.mountPoint = mountPoint this.parent = parent - this.scriptFactory = null + this._scriptFactory = scriptFactory this.initParameters = initParameters ?: [:] this._scriptFactoryArgs = scriptFactory.toExternalRepresentation() } def getScriptFactoryArgs() { - if(_scriptFactoryArgs == null) - return scriptFactory.toExternalRepresentation() - else - _scriptFactoryArgs + _scriptFactoryArgs } - ScriptFactory getScriptFactory(ScriptFactoryFactory scriptFactoryFactory) + ScriptFactory getScriptFactory() { - // for backward compatibility - if(_scriptFactoryArgs == null) - return scriptFactory - else - return scriptFactoryFactory.createScriptFactory(_scriptFactoryArgs) + return _scriptFactory + } + + void setScriptFactory(ScriptFactory factory) + { + throw new UnsupportedOperationException("read only") } public String toString() { - return "[mountPoint: ${mountPoint}, parent: ${parent}, scriptFactory: ${scriptFactory}, scriptFactoryArgs: ${_scriptFactoryArgs}, initParameters: ${initParameters}]"; + return "[mountPoint: ${mountPoint}, parent: ${parent}, scriptFactoryArgs: ${scriptFactoryArgs}, initParameters: ${initParameters}]"; } def toExternalRepresentation() { return [mountPoint: mountPoint, parent: parent, - scriptFactory: scriptFactory?.toExternalRepresentation() ?: _scriptFactoryArgs, + scriptFactory: scriptFactoryArgs, initParameters: initParameters] } @@ -90,7 +90,6 @@ class ScriptDefinition implements Serializable, Externable if(that.mountPoint != this.mountPoint) return false; if(that.parent != this.parent) return false; - if(that.scriptFactory != this.scriptFactory) return false; if(that._scriptFactoryArgs != this._scriptFactoryArgs) return false; if(that.initParameters != this.initParameters) return false; @@ -103,8 +102,7 @@ class ScriptDefinition implements Serializable, Externable result = (mountPoint ? mountPoint.hashCode() : 0); result = 31 * result + (parent ? parent.hashCode() : 0); - result = 31 * result + (scriptFactory ? scriptFactory.hashCode() : 0); - result = 31 * result + (scriptFactoryArgs ? scriptFactoryArgs.hashCode() : 0); + result = 31 * result + (_scriptFactoryArgs ? _scriptFactoryArgs.hashCode() : 0); result = 31 * result + (initParameters ? initParameters.hashCode() : 0); return result; } diff --git a/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/script/ScriptFactory.groovy b/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/script/ScriptFactory.groovy index 470d31c3..27f96c36 100644 --- a/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/script/ScriptFactory.groovy +++ b/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/script/ScriptFactory.groovy @@ -1,5 +1,6 @@ /* * Copyright (c) 2010-2010 LinkedIn, Inc + * Portions Copyright (c) 2013 Yan Pujante * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -28,6 +29,11 @@ interface ScriptFactory */ def createScript(ScriptConfig scriptConfig) + /** + * Called to destroy the script + */ + void destroyScript(ScriptConfig scriptConfig) + /** * @returns an external representation of the factory */ diff --git a/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/script/ScriptManagerImpl.groovy b/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/script/ScriptManagerImpl.groovy index 7dc13aa1..8c3d9de1 100644 --- a/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/script/ScriptManagerImpl.groovy +++ b/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/script/ScriptManagerImpl.groovy @@ -1,6 +1,6 @@ /* * Copyright (c) 2010-2010 LinkedIn, Inc - * Portions Copyright (c) 2011 Yan Pujante + * Portions Copyright (c) 2011-2013 Yan Pujante * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -147,7 +147,7 @@ def class ScriptManagerImpl implements ScriptManager try { - childScript = sd.getScriptFactory(scriptFactoryFactory).createScript(scriptConfig) + childScript = sd.scriptFactory.createScript(scriptConfig) } catch(Exception e) { @@ -325,6 +325,12 @@ def class ScriptManagerImpl implements ScriptManager // we clean up the temp space node.shell.rmdirs(node.shell.fileSystem.tmpRoot) + def scriptConfig = new ScriptConfig(shell: node.shell, + agentContext: agentContext) + + // give a chance to the factory to "release" some resources + node.scriptDefinition.scriptFactory.destroyScript(scriptConfig) + removeScriptNode(mountPoint) node.log.info("uninstalled") diff --git a/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/zookeeper/ZooKeeperStorage.groovy b/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/zookeeper/ZooKeeperStorage.groovy index 4a373f51..2e7cda71 100644 --- a/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/zookeeper/ZooKeeperStorage.groovy +++ b/agent/org.linkedin.glu.agent-impl/src/main/groovy/org/linkedin/glu/agent/impl/zookeeper/ZooKeeperStorage.groovy @@ -1,6 +1,6 @@ /* * Copyright (c) 2010-2010 LinkedIn, Inc - * Portions Copyright (c) 2011 Yan Pujante + * Portions Copyright (c) 2011-2013 Yan Pujante * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -85,6 +85,11 @@ class ZooKeeperStorage implements WriteOnlyStorage log.warn("Call ignored ${closure.toString()} due to unexpected exception", e) } + log.warn("Call ignored ${closure.toString()}: zookeeper is not connected") + + if(log.isDebugEnabled()) + log.debug("zkSafe => Call ignored due to zk not connected!", new Exception("where am I?")) + return null } diff --git a/agent/org.linkedin.glu.agent-impl/src/perf-test/groovy/test/agent/perf/TestPerfTimers.groovy b/agent/org.linkedin.glu.agent-impl/src/perf-test/groovy/test/agent/perf/TestPerfTimers.groovy index e6ddbdd7..76aa0edd 100644 --- a/agent/org.linkedin.glu.agent-impl/src/perf-test/groovy/test/agent/perf/TestPerfTimers.groovy +++ b/agent/org.linkedin.glu.agent-impl/src/perf-test/groovy/test/agent/perf/TestPerfTimers.groovy @@ -1,5 +1,6 @@ /* * Copyright (c) 2010-2010 LinkedIn, Inc + * Portions Copyright (c) 2013 Yan Pujante * * Licensed under the Apache License, Version 2.0 (the "License"); you may not * use this file except in compliance with the License. You may obtain a copy of @@ -18,7 +19,7 @@ package test.agent.perf /** * Example of running the performance test: - * groovy groovy/test/agent/perf/TestPerfTimers.groovy /Volumes/raptor/deployment/glu/dist/agent/client/bin/gluc.sh resources/TimersTestScript.groovy + * groovy groovy/test/agent/perf/TestPerfTimers.groovy /export/content/glu/org.linkedin.glu.packaging-all-xxx/bin/agent-cli.sh resources/TimersTestScript.groovy * @author ypujante@linkedin.com */ class TestPerfTimers { diff --git a/build.gradle b/build.gradle index 40977783..4c533e06 100644 --- a/build.gradle +++ b/build.gradle @@ -41,7 +41,7 @@ buildscript { } dependencies { - classpath 'org.linkedin:org.linkedin.gradle-plugins:1.5.glu47.3' + classpath 'org.linkedin:org.linkedin.gradle-plugins:1.6.0' } } @@ -75,6 +75,13 @@ allprojects { apply plugin: 'idea' group = spec.group version = spec.version + + // this is to force JUnit to a consistent version (zookeeper is bringing a different one!) + configurations.all { + resolutionStrategy { + force spec.external.junit + } + } } def ideaCopyright = """ diff --git a/project-spec.groovy b/project-spec.groovy index 24d78c37..f6ffc4b7 100644 --- a/project-spec.groovy +++ b/project-spec.groovy @@ -26,8 +26,8 @@ spec = [ grails: '2.2.1', groovy: '2.0.7', jetty: '8.1.10.v20130312', // '9.0.0.v20130308' (cannot use 9 -> requires jdk 1.7) - linkedinUtils: '1.8.glu47.2', - linkedinZookeeper: '1.5.glu47.1', + linkedinUtils: '1.8.glu47.3', + linkedinZookeeper: '1.5.glu47.2', restlet: '2.1.2', sigar: '1.6.4', slf4j: '1.6.2' // to be compatible with grails 2.2.1 diff --git a/provisioner/org.linkedin.glu.provisioner-core/src/main/groovy/org/linkedin/glu/provisioner/core/model/SystemFilterBuilder.groovy b/provisioner/org.linkedin.glu.provisioner-core/src/main/groovy/org/linkedin/glu/provisioner/core/model/SystemFilterBuilder.groovy index da962c03..3b6d1960 100644 --- a/provisioner/org.linkedin.glu.provisioner-core/src/main/groovy/org/linkedin/glu/provisioner/core/model/SystemFilterBuilder.groovy +++ b/provisioner/org.linkedin.glu.provisioner-core/src/main/groovy/org/linkedin/glu/provisioner/core/model/SystemFilterBuilder.groovy @@ -76,17 +76,25 @@ public class SystemFilterBuilder def builder = new SystemFilterBuilder(filter: new LogicAndSystemFilterChain()) - Script script = new GroovyShell(new BindingBuilder(systemFilterBuilder: builder)).parse(dsl) + def shell = new GroovyShell(new BindingBuilder(systemFilterBuilder: builder)) + try + { + Script script = shell.parse(dsl) - script.metaClass = createEMC(script.class) { ExpandoMetaClass emc -> - ['and', 'or', 'not', 'c'].each { method -> - emc."${method}" = { Closure cl -> - builder."${method}"(cl) + script.metaClass = createEMC(script.class) { ExpandoMetaClass emc -> + ['and', 'or', 'not', 'c'].each { method -> + emc."${method}" = { Closure cl -> + builder."${method}"(cl) + } } } - } - script.run() + script.run() + } + finally + { + shell.resetLoadedClasses() + } return builder.optimizeFilter(builder.filter) }