diff --git a/src/main/java/org/apache/sysml/api/DMLScript.java b/src/main/java/org/apache/sysml/api/DMLScript.java index 16d8986437e..293a8109abf 100644 --- a/src/main/java/org/apache/sysml/api/DMLScript.java +++ b/src/main/java/org/apache/sysml/api/DMLScript.java @@ -376,56 +376,56 @@ else if (debug.equalsIgnoreCase("trace")) // (core compilation and execute) //////// - /** - * The running body of DMLScript execution. This method should be called after execution properties have been correctly set, - * and customized parameters have been put into _argVals - * - * @param dmlScriptStr DML script string - * @param fnameOptConfig configuration file - * @param argVals map of argument values - * @param allArgs arguments - * @param scriptType type of script (DML or PyDML) - * @throws IOException if IOException occurs - */ - private static void execute(String dmlScriptStr, String fnameOptConfig, Map argVals, String[] allArgs, ScriptType scriptType) - throws IOException - { - SCRIPT_TYPE = scriptType; - - //print basic time and environment info - printStartExecInfo( dmlScriptStr ); - - //Step 1: parse configuration files & write any configuration specific global variables - DMLConfig dmlconf = DMLConfig.readConfigurationFile(fnameOptConfig); - ConfigurationManager.setGlobalConfig(dmlconf); - CompilerConfig cconf = OptimizerUtils.constructCompilerConfig(dmlconf); - ConfigurationManager.setGlobalConfig(cconf); - LOG.debug("\nDML config: \n" + dmlconf.getConfigInfo()); - - setGlobalFlags(dmlconf); - Program rtprog = ScriptExecutorUtils.compileRuntimeProgram(dmlScriptStr, argVals, allArgs, - scriptType, dmlconf, SystemMLAPI.DMLScript); - List gCtxs = ConfigurationManager.getDMLOptions().gpu ? GPUContextPool.getAllGPUContexts() : null; - - //double costs = CostEstimationWrapper.getTimeEstimate(rtprog, ExecutionContextFactory.createContext()); - //System.out.println("Estimated costs: "+costs); - - //Step 10: execute runtime program - ExecutionContext ec = null; - try { - ec = ScriptExecutorUtils.executeRuntimeProgram( - rtprog, dmlconf, ConfigurationManager.isStatistics() ? - ConfigurationManager.getDMLOptions().getStatisticsMaxHeavyHitters() : 0, - new LocalVariableMap(), null, SystemMLAPI.DMLScript, gCtxs); - } - finally { - if(ec != null && ec instanceof SparkExecutionContext) - ((SparkExecutionContext) ec).close(); - LOG.info("END DML run " + getDateTime() ); - //cleanup scratch_space and all working dirs - cleanupHadoopExecution( dmlconf ); - } - } + /** + * The running body of DMLScript execution. This method should be called after execution properties have been correctly set, + * and customized parameters have been put into _argVals + * + * @param dmlScriptStr DML script string + * @param fnameOptConfig configuration file + * @param argVals map of argument values + * @param allArgs arguments + * @param scriptType type of script (DML or PyDML) + * @throws IOException if IOException occurs + */ + private static void execute(String dmlScriptStr, String fnameOptConfig, Map argVals, String[] allArgs, ScriptType scriptType) + throws IOException + { + SCRIPT_TYPE = scriptType; + + //print basic time and environment info + printStartExecInfo( dmlScriptStr ); + + //Step 1: parse configuration files & write any configuration specific global variables + DMLConfig dmlconf = DMLConfig.readConfigurationFile(fnameOptConfig); + ConfigurationManager.setGlobalConfig(dmlconf); + CompilerConfig cconf = OptimizerUtils.constructCompilerConfig(dmlconf); + ConfigurationManager.setGlobalConfig(cconf); + LOG.debug("\nDML config: \n" + dmlconf.getConfigInfo()); + + setGlobalFlags(dmlconf); + Program rtprog = ScriptExecutorUtils.compileRuntimeProgram(dmlScriptStr, argVals, allArgs, + scriptType, dmlconf, SystemMLAPI.DMLScript); + List gCtxs = ConfigurationManager.getDMLOptions().gpu ? GPUContextPool.getAllGPUContexts() : null; + + //double costs = CostEstimationWrapper.getTimeEstimate(rtprog, ExecutionContextFactory.createContext()); + //System.out.println("Estimated costs: "+costs); + + //Step 10: execute runtime program + ExecutionContext ec = null; + try { + ec = ScriptExecutorUtils.executeRuntimeProgram( + rtprog, ConfigurationManager.isStatistics() ? + ConfigurationManager.getDMLOptions().getStatisticsMaxHeavyHitters() : 0, + new LocalVariableMap(), null, SystemMLAPI.DMLScript, gCtxs); + } + finally { + if(ec != null && ec instanceof SparkExecutionContext) + ((SparkExecutionContext) ec).close(); + LOG.info("END DML run " + getDateTime() ); + //cleanup scratch_space and all working dirs + cleanupHadoopExecution( dmlconf ); + } + } /** * Sets the global flags in DMLScript based on user provided configuration @@ -547,8 +547,8 @@ private static void checkSecuritySetup(DMLConfig config) //determine security states boolean flagDiffUser = !( taskController.equals("org.apache.hadoop.mapred.LinuxTaskController") //runs map/reduce tasks as the current user - || localMode // run in the same JVM anyway - || groupNames.contains( ttGroupName) ); //user in task tracker group + || localMode // run in the same JVM anyway + || groupNames.contains( ttGroupName) ); //user in task tracker group boolean flagLocalFS = fsURI==null || fsURI.getScheme().equals("file"); boolean flagSecurity = perm.equals("yes"); diff --git a/src/main/java/org/apache/sysml/api/ScriptExecutorUtils.java b/src/main/java/org/apache/sysml/api/ScriptExecutorUtils.java index b1b57355ae2..e2e76cbeea9 100644 --- a/src/main/java/org/apache/sysml/api/ScriptExecutorUtils.java +++ b/src/main/java/org/apache/sysml/api/ScriptExecutorUtils.java @@ -116,7 +116,7 @@ public static Program compileRuntimeProgram(String script, Map ns boolean init) { DMLScript.SCRIPT_TYPE = scriptType; - Program rtprog = null; + Program rtprog; if (ConfigurationManager.isGPU() && !IS_JCUDA_AVAILABLE) throw new RuntimeException("Incorrect usage: Cannot use the GPU backend without JCuda libraries. Hint: Include systemml-*-extra.jar (compiled using mvn package -P distribution) into the classpath."); @@ -161,7 +161,7 @@ else if (!ConfigurationManager.isGPU() && ConfigurationManager.isForcedGPU()) //init working directories (before usage by following compilation steps) if(api != SystemMLAPI.JMLC) - if ((api == SystemMLAPI.MLContext && init) || api != SystemMLAPI.MLContext) + if (api != SystemMLAPI.MLContext || init) DMLScript.initHadoopExecution( dmlconf ); @@ -222,8 +222,9 @@ else if(api == SystemMLAPI.MLContext) { ExplainCounts counts = Explain.countDistributedOperations(rtprog); Statistics.resetNoOfCompiledJobs( counts.numJobs ); //explain plan of program (hops or runtime) - if( DMLScript.EXPLAIN != ExplainType.NONE ) - System.out.println(Explain.display(prog, rtprog, DMLScript.EXPLAIN, counts)); + if( ConfigurationManager.getDMLOptions().explainType != ExplainType.NONE ) + System.out.println( + Explain.display(prog, rtprog, ConfigurationManager.getDMLOptions().explainType, counts)); Statistics.stopCompileTimer(); } @@ -232,9 +233,6 @@ else if(api == SystemMLAPI.MLContext) { // don't chain ParseException (for cleaner error output) throw pe; } - catch(IOException ex) { - throw new DMLException(ex); - } catch(Exception ex) { throw new DMLException(ex); } @@ -248,8 +246,6 @@ else if(api == SystemMLAPI.MLContext) { * * @param rtprog * runtime program - * @param dmlconf - * dml configuration * @param statisticsMaxHeavyHitters * maximum number of statistics to print * @param symbolTable @@ -262,7 +258,7 @@ else if(api == SystemMLAPI.MLContext) { * list of GPU contexts * @return execution context */ - public static ExecutionContext executeRuntimeProgram(Program rtprog, DMLConfig dmlconf, int statisticsMaxHeavyHitters, + public static ExecutionContext executeRuntimeProgram(Program rtprog, int statisticsMaxHeavyHitters, LocalVariableMap symbolTable, HashSet outputVariables, SystemMLAPI api, List gCtxs) { boolean exceptionThrown = false; @@ -299,7 +295,7 @@ public static ExecutionContext executeRuntimeProgram(Program rtprog, DMLConfig d if(outputVariables != null) { for(String outVar : outputVariables) { Data data = ec.getVariable(outVar); - if(data != null && data instanceof MatrixObject) { + if(data instanceof MatrixObject) { for(GPUContext gCtx : ec.getGPUContexts()) { GPUObject gpuObj = ((MatrixObject)data).getGPUObject(gCtx); if(gpuObj != null && gpuObj.isDirty()) { @@ -341,4 +337,4 @@ public static ExecutionContext executeRuntimeProgram(Program rtprog, DMLConfig d return ec; } -} \ No newline at end of file +} diff --git a/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java b/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java index 522409779df..edbd5bfdc46 100644 --- a/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java +++ b/src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java @@ -32,7 +32,6 @@ import org.apache.sysml.api.DMLException; import org.apache.sysml.api.ScriptExecutorUtils; import org.apache.sysml.api.ScriptExecutorUtils.SystemMLAPI; -import org.apache.sysml.api.DMLScript; import org.apache.sysml.conf.CompilerConfig; import org.apache.sysml.conf.ConfigurationManager; import org.apache.sysml.conf.DMLConfig; @@ -47,8 +46,6 @@ import org.apache.sysml.runtime.controlprogram.Program; import org.apache.sysml.runtime.controlprogram.caching.FrameObject; import org.apache.sysml.runtime.controlprogram.caching.MatrixObject; -import org.apache.sysml.runtime.controlprogram.context.ExecutionContext; -import org.apache.sysml.runtime.controlprogram.context.ExecutionContextFactory; import org.apache.sysml.runtime.instructions.cp.BooleanObject; import org.apache.sysml.runtime.instructions.cp.Data; import org.apache.sysml.runtime.instructions.cp.DoubleObject; @@ -100,6 +97,9 @@ private PreparedScript(PreparedScript that) { _inVarReuse = new HashMap<>(that._inVarReuse); _dmlconf = that._dmlconf; _cconf = that._cconf; + _isStatisticsEnabled = that._isStatisticsEnabled; + _gatherMemStats = that._gatherMemStats; + _gpuCtx = that._gpuCtx; } /** @@ -463,7 +463,7 @@ public ResultVariables executeScript() { //create and populate execution context ScriptExecutorUtils.executeRuntimeProgram( - _prog, _dmlconf, ConfigurationManager.isStatistics() ? + _prog, ConfigurationManager.isStatistics() ? ConfigurationManager.getDMLOptions().getStatisticsMaxHeavyHitters() : 0, _vars, _outVarnames, SystemMLAPI.JMLC, _gpuCtx); diff --git a/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java b/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java index 06778b35b08..7bda30696d2 100644 --- a/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java +++ b/src/main/java/org/apache/sysml/api/mlcontext/ScriptExecutor.java @@ -19,7 +19,6 @@ package org.apache.sysml.api.mlcontext; -import java.io.IOException; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -31,21 +30,15 @@ import org.apache.sysml.api.DMLScript.RUNTIME_PLATFORM; import org.apache.sysml.api.ScriptExecutorUtils; import org.apache.sysml.api.ScriptExecutorUtils.SystemMLAPI; -import org.apache.sysml.api.jmlc.JMLCUtils; import org.apache.sysml.api.mlcontext.MLContext.ExecutionType; import org.apache.sysml.api.mlcontext.MLContext.ExplainLevel; import org.apache.sysml.conf.CompilerConfig; import org.apache.sysml.conf.ConfigurationManager; import org.apache.sysml.conf.DMLConfig; import org.apache.sysml.conf.DMLOptions; -import org.apache.sysml.hops.HopsException; import org.apache.sysml.hops.OptimizerUtils; -import org.apache.sysml.hops.rewrite.ProgramRewriter; -import org.apache.sysml.hops.rewrite.RewriteRemovePersistentReadWrite; -import org.apache.sysml.lops.LopsException; import org.apache.sysml.parser.DMLProgram; import org.apache.sysml.parser.DMLTranslator; -import org.apache.sysml.parser.LanguageException; import org.apache.sysml.parser.ParseException; import org.apache.sysml.parser.ParserFactory; import org.apache.sysml.parser.ParserWrapper; @@ -56,7 +49,6 @@ import org.apache.sysml.runtime.instructions.gpu.context.GPUContext; import org.apache.sysml.runtime.instructions.gpu.context.GPUContextPool; import org.apache.sysml.utils.Explain; -import org.apache.sysml.utils.Explain.ExplainCounts; import org.apache.sysml.utils.Explain.ExplainType; import org.apache.sysml.utils.Statistics; @@ -102,7 +94,6 @@ public class ScriptExecutor { protected DMLConfig config; protected DMLProgram dmlProgram; - protected DMLTranslator dmlTranslator; protected Program runtimeProgram; protected ExecutionContext executionContext; protected Script script; @@ -138,80 +129,6 @@ public ScriptExecutor(DMLConfig config) { ConfigurationManager.setGlobalConfig(config); } - /** - * Construct DAGs of high-level operators (HOPs) for each block of - * statements. - */ - protected void constructHops() { - try { - dmlTranslator.constructHops(dmlProgram); - } catch (LanguageException | ParseException e) { - throw new MLContextException("Exception occurred while constructing HOPS (high-level operators)", e); - } - } - - /** - * Apply static rewrites, perform intra-/inter-procedural analysis to - * propagate size information into functions, apply dynamic rewrites, and - * compute memory estimates for all HOPs. - */ - protected void rewriteHops() { - try { - dmlTranslator.rewriteHopsDAG(dmlProgram); - } catch (LanguageException | HopsException | ParseException | DMLRuntimeException e) { - throw new MLContextException("Exception occurred while rewriting HOPS (high-level operators)", e); - } - } - - /** - * Output a description of the program to standard output. - */ - protected void showExplanation() { - if (!explain) - return; - - try { - ExplainType explainType = (explainLevel != null) ? explainLevel.getExplainType() : ExplainType.RUNTIME; - System.out.println(Explain.display(dmlProgram, runtimeProgram, explainType, null)); - } catch (Exception e) { - throw new MLContextException("Exception occurred while explaining dml program", e); - } - } - - /** - * Construct DAGs of low-level operators (LOPs) based on the DAGs of - * high-level operators (HOPs). - */ - protected void constructLops() { - try { - dmlTranslator.constructLops(dmlProgram); - } catch (ParseException | LanguageException | HopsException | LopsException e) { - throw new MLContextException("Exception occurred while constructing LOPS (low-level operators)", e); - } - } - - /** - * Create runtime program. For each namespace, translate function statement - * blocks into function program blocks and add these to the runtime program. - * For each top-level block, add the program block to the runtime program. - */ - protected void generateRuntimeProgram() { - try { - runtimeProgram = dmlTranslator.getRuntimeProgram(dmlProgram, config); - } catch (LanguageException | DMLRuntimeException | LopsException | IOException | HopsException e) { - throw new MLContextException("Exception occurred while generating runtime program", e); - } - } - - /** - * Count the number of compiled MR Jobs/Spark Instructions in the runtime - * program and set this value in the statistics. - */ - protected void countCompiledMRJobsAndSparkInstructions() { - ExplainCounts counts = Explain.countDistributedOperations(runtimeProgram); - Statistics.resetNoOfCompiledJobs(counts.numJobs); - } - /** * Set the global flags (for example: statistics, gpu, etc). */ @@ -255,25 +172,6 @@ public void compile(Script script) { /** * Compile a DML or PYDML script. This will help analysis of DML programs * that have dynamic recompilation flag set to false without actually executing it. - * - * This is broken down into the following - * primary methods: - * - *
    - *
  1. {@link #setup(Script)}
    2. - *
  2. {@link #parseScript()}
    3. - *
  3. {@link #liveVariableAnalysis()}
    4. - *
  4. {@link #validateScript()}
    5. - *
  5. {@link #constructHops()}
    6. - *
  6. {@link #rewriteHops()}
    7. - *
  7. {@link #rewritePersistentReadsAndWrites()}
    8. - *
  8. {@link #constructLops()}
    9. - *
  9. {@link #generateRuntimeProgram()}
    10. - *
  10. {@link #showExplanation()}
    11. - *
  11. {@link #countCompiledMRJobsAndSparkInstructions()}
    12. - *
  12. {@link #initializeCachingAndScratchSpace()}
    13. - *
  13. {@link #cleanupRuntimeProgram()}
    14. - *
* * @param script * the DML or PYDML script to compile @@ -321,8 +219,8 @@ public MLResults execute(Script script) { .convertInputParametersForParser(script.getInputParameters(), script.getScriptType()); Explain.ExplainType explainType = Explain.ExplainType.NONE; - if(explain && explainLevel != null) { - explainType = explainLevel.getExplainType(); + if(explain) { + explainType = (explainLevel == null) ? Explain.ExplainType.RUNTIME : explainLevel.getExplainType(); } RUNTIME_PLATFORM rtplatform = DMLOptions.defaultOptions.execMode; if(executionType != null) { @@ -332,14 +230,14 @@ public MLResults execute(Script script) { statistics, statisticsMaxHeavyHitters, false, explainType, rtplatform, gpu, forceGPU, script.getScriptType(), DMLScript.DML_FILE_PATH_ANTLR_PARSER, script.getScriptExecutionString())); - + // main steps in script execution compile(script); try { - executionContext = ScriptExecutorUtils.executeRuntimeProgram(getRuntimeProgram(), getConfig(), + executionContext = ScriptExecutorUtils.executeRuntimeProgram(getRuntimeProgram(), statistics ? statisticsMaxHeavyHitters : 0, script.getSymbolTable(), - new HashSet(getScript().getOutputVariables()), SystemMLAPI.MLContext, gCtxs); + new HashSet<>(getScript().getOutputVariables()), SystemMLAPI.MLContext, gCtxs); } catch (DMLRuntimeException e) { throw new MLContextException("Exception occurred while executing runtime program", e); } finally { @@ -410,41 +308,6 @@ protected void restoreInputsInSymbolTable() { } } - /** - * If {@code maintainSymbolTable} is true, delete all 'remove variable' - * instructions so as to maintain the values in the symbol table, which are - * useful when working interactively in an environment such as the Spark - * Shell. Otherwise, only delete 'remove variable' instructions for - * registered outputs. - */ - protected void cleanupRuntimeProgram() { - if (maintainSymbolTable) { - MLContextUtil.deleteRemoveVariableInstructions(runtimeProgram); - } else { - JMLCUtils.cleanupRuntimeProgram(runtimeProgram, (script.getOutputVariables() == null) ? new String[0] - : script.getOutputVariables().toArray(new String[0])); - } - } - - /** - * Check security, create scratch space, cleanup working directories, - * initialize caching, and reset statistics. - */ - protected void initializeCachingAndScratchSpace() { - if (!init) - return; - - try { - DMLScript.initHadoopExecution(config); - } catch (ParseException e) { - throw new MLContextException("Exception occurred initializing caching and scratch space", e); - } catch (DMLRuntimeException e) { - throw new MLContextException("Exception occurred initializing caching and scratch space", e); - } catch (IOException e) { - throw new MLContextException("Exception occurred initializing caching and scratch space", e); - } - } - /** * Parse the script into an ANTLR parse tree, and convert this parse tree * into a SystemML program. Parsing includes lexical/syntactic analysis. @@ -460,29 +323,6 @@ protected void parseScript() { } } - /** - * Replace persistent reads and writes with transient reads and writes in - * the symbol table. - */ - protected void rewritePersistentReadsAndWrites() { - LocalVariableMap symbolTable = script.getSymbolTable(); - if (symbolTable != null) { - String[] inputs = (script.getInputVariables() == null) ? new String[0] - : script.getInputVariables().toArray(new String[0]); - String[] outputs = (script.getOutputVariables() == null) ? new String[0] - : script.getOutputVariables().toArray(new String[0]); - RewriteRemovePersistentReadWrite rewrite = new RewriteRemovePersistentReadWrite(inputs, outputs, - script.getSymbolTable()); - ProgramRewriter programRewriter = new ProgramRewriter(rewrite); - try { - programRewriter.rewriteProgramHopDAGs(dmlProgram); - } catch (LanguageException | HopsException e) { - throw new MLContextException("Exception occurred while rewriting persistent reads and writes", e); - } - } - - } - /** * Set the SystemML configuration properties. * @@ -494,50 +334,6 @@ public void setConfig(DMLConfig config) { ConfigurationManager.setGlobalConfig(config); } - /** - * Liveness analysis is performed on the program, obtaining sets of live-in - * and live-out variables by forward and backward passes over the program. - */ - protected void liveVariableAnalysis() { - try { - dmlTranslator = new DMLTranslator(dmlProgram); - dmlTranslator.liveVariableAnalysis(dmlProgram); - } catch (DMLRuntimeException e) { - throw new MLContextException("Exception occurred during live variable analysis", e); - } catch (LanguageException e) { - throw new MLContextException("Exception occurred during live variable analysis", e); - } - } - - /** - * Semantically validate the program's expressions, statements, and - * statement blocks in a single recursive pass over the program. Constant - * and size propagation occurs during this step. - */ - protected void validateScript() { - try { - dmlTranslator.validateParseTree(dmlProgram); - } catch (LanguageException | ParseException e) { - throw new MLContextException("Exception occurred while validating script", e); - } - } - - /** - * Check that the Script object has a type (DML or PYDML) and a string - * representing the content of the Script. - */ - protected void checkScriptHasTypeAndString() { - if (script == null) { - throw new MLContextException("Script is null"); - } else if (script.getScriptType() == null) { - throw new MLContextException("ScriptType (DML or PYDML) needs to be specified"); - } else if (script.getScriptString() == null) { - throw new MLContextException("Script string is null"); - } else if (StringUtils.isBlank(script.getScriptString())) { - throw new MLContextException("Script string is blank"); - } - } - /** * Obtain the program * @@ -547,15 +343,6 @@ public DMLProgram getDmlProgram() { return dmlProgram; } - /** - * Obtain the translator - * - * @return the translator - */ - public DMLTranslator getDmlTranslator() { - return dmlTranslator; - } - /** * Obtain the runtime program * @@ -591,9 +378,7 @@ public Script getScript() { * {@code true} if explanation should be output, {@code false} * otherwise */ - public void setExplain(boolean explain) { - this.explain = explain; - } + public void setExplain(boolean explain) { this.explain = explain; } /** * Whether or not statistics about the DML/PYDML program should be output to @@ -664,8 +449,7 @@ public void setExplainLevel(ExplainLevel explainLevel) { if (explainLevel == null) { DMLScript.EXPLAIN = ExplainType.NONE; } else { - ExplainType explainType = explainLevel.getExplainType(); - DMLScript.EXPLAIN = explainType; + DMLScript.EXPLAIN = explainLevel.getExplainType(); } } @@ -717,4 +501,4 @@ public void setExecutionType(ExecutionType executionType) { ConfigurationManager.getDMLOptions().setExecutionMode(executionType.getRuntimePlatform()); this.executionType = executionType; } -} \ No newline at end of file +} diff --git a/src/main/java/org/apache/sysml/conf/ConfigurationManager.java b/src/main/java/org/apache/sysml/conf/ConfigurationManager.java index 08d929ef994..c10b3a53777 100644 --- a/src/main/java/org/apache/sysml/conf/ConfigurationManager.java +++ b/src/main/java/org/apache/sysml/conf/ConfigurationManager.java @@ -37,10 +37,10 @@ public class ConfigurationManager { /** Global cached job conf for read-only operations */ - private static JobConf _rJob = null; + private static JobConf _rJob; /** Global DML configuration (read or defaults) */ - private static DMLConfig _dmlconf = null; + private static DMLConfig _dmlconf; /** Local DML configuration for thread-local config updates */ private static ThreadLocalDMLConfig _ldmlconf = new ThreadLocalDMLConfig(); @@ -52,7 +52,7 @@ public class ConfigurationManager private static ThreadLocalDMLOptions _ldmlOptions = new ThreadLocalDMLOptions(); /** Global compiler configuration (defaults) */ - private static CompilerConfig _cconf = null; + private static CompilerConfig _cconf; /** Local compiler configuration for thead-local config updates */ private static ThreadLocalCompilerConfig _lcconf = new ThreadLocalCompilerConfig(); @@ -208,7 +208,7 @@ public static CompilerConfig getCompilerConfig() { */ public static boolean getCompilerConfigFlag(ConfigType key) { CompilerConfig cconf = getCompilerConfig(); - return (cconf!=null) ? cconf.getBool(key) : false; + return (cconf!=null) && cconf.getBool(key); } ///////////////////////////////////// diff --git a/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java b/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java index 220f3e51448..58a869493b9 100644 --- a/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java +++ b/src/main/java/org/apache/sysml/runtime/controlprogram/LocalVariableMap.java @@ -26,7 +26,6 @@ import java.util.Set; import java.util.StringTokenizer; -import org.apache.sysml.api.DMLScript; import org.apache.sysml.conf.ConfigurationManager; import org.apache.sysml.runtime.controlprogram.caching.CacheableData; import org.apache.sysml.runtime.util.ProgramConverter; diff --git a/src/test/java/org/apache/sysml/test/gpu/JMLCTests.java b/src/test/java/org/apache/sysml/test/gpu/JMLCTests.java index d77d794aed7..be02976acaf 100644 --- a/src/test/java/org/apache/sysml/test/gpu/JMLCTests.java +++ b/src/test/java/org/apache/sysml/test/gpu/JMLCTests.java @@ -28,99 +28,97 @@ public class JMLCTests extends GPUTests { - static class ScriptContainer { - String dml; - String[] inputVarNames; - } - - @Test - public void testJMLC() { - try { - Connection conn = new Connection(); - - int numMatrices = 10; - int matrixNumRows = 100; - int numScriptInvocations = 10; - - ScriptContainer SC = generateDMLScript(numMatrices); - - PreparedScript script = conn.prepareScript( - SC.dml, SC.inputVarNames, new String[] { "Z" }, true, true, 0); - - // execute the script without pinning input matrices between invocations - executeDMLScript(script, numScriptInvocations, matrixNumRows, numMatrices, false); - - // execute the script while pinning input matrices between invocations - executeDMLScript(script, numScriptInvocations, matrixNumRows, numMatrices, true); - } catch (Exception e) { - Assert.fail("An unexpected exception occurred: " + e.getMessage()); - } - } - - // Generates a simple synthetic DML script which multiplies a sequence of square matrices. - // I.e. Z = X %*% W1 %*% W2 %*% W3 ... - // numMatrices determines the number of matrices in the sequences. The size of the matrices can be set - // in executeDMLScript - static ScriptContainer generateDMLScript(int numMatrices) { - ScriptContainer SC = new ScriptContainer(); - String[] inputVarNames = new String[numMatrices + 1]; - inputVarNames[0] = "x"; - - StringBuilder dml = new StringBuilder("x = read(\"/tmp/X.mtx\", rows=-1, cols=-1)\n"); - for (int ix=0; ix 1)\n print(as.scalar(Z[1,1]))\n"); - - SC.dml = dml.toString(); - SC.inputVarNames = inputVarNames; - - return SC; - } - - // Executes a PreparedScript generated by generateDMLScript. The parameter n determines the - // number of times the script is invoked. The parameter rows controls the shape of the matrices. - // Set this parameter larger to use more memory. The parameter numMatrices must be set to the same value as - // in generateDMLScript. The parameter pinWeights controls whether weight matrices should be - // pinned in memory between script invocations. - static void executeDMLScript(PreparedScript script, int n, int rows, int numMatrices, boolean pinWeights) { - for (int ix=0; ix sparsity) { - continue; - } - matrix[i][j] = (random.nextDouble() * (max - min) + min); - } - } - return matrix; - } + static class ScriptContainer { + String dml; + String[] inputVarNames; + } + + @Test + public void testJMLC() { + try { + Connection conn = new Connection(); + + int numMatrices = 10; + int matrixNumRows = 100; + int numScriptInvocations = 10; + + ScriptContainer SC = generateDMLScript(numMatrices); + + PreparedScript script = conn.prepareScript( + SC.dml, SC.inputVarNames, new String[] { "Z" }, true, true, 0); + + // execute the script without pinning input matrices between invocations + executeDMLScript(script, numScriptInvocations, matrixNumRows, numMatrices, false); + + // execute the script while pinning input matrices between invocations + executeDMLScript(script, numScriptInvocations, matrixNumRows, numMatrices, true); + } catch (Exception e) { + Assert.fail("An unexpected exception occurred: " + e.getMessage()); + } + } + + // Generates a simple synthetic DML script which multiplies a sequence of square matrices. + // I.e. Z = X %*% W1 %*% W2 %*% W3 ... + // numMatrices determines the number of matrices in the sequences. The size of the matrices can be set + // in executeDMLScript + private static ScriptContainer generateDMLScript(int numMatrices) { + ScriptContainer SC = new ScriptContainer(); + String[] inputVarNames = new String[numMatrices + 1]; + inputVarNames[0] = "x"; + + StringBuilder dml = new StringBuilder("x = read(\"/tmp/X.mtx\", rows=-1, cols=-1)\n"); + for (int ix=0; ix 1)\n print(as.scalar(Z[1,1]))\n"); + + SC.dml = dml.toString(); + SC.inputVarNames = inputVarNames; + + return SC; + } + + // Executes a PreparedScript generated by generateDMLScript. The parameter n determines the + // number of times the script is invoked. The parameter rows controls the shape of the matrices. + // Set this parameter larger to use more memory. The parameter numMatrices must be set to the same value as + // in generateDMLScript. The parameter pinWeights controls whether weight matrices should be + // pinned in memory between script invocations. + private static void executeDMLScript(PreparedScript script, int n, int rows, int numMatrices, boolean pinWeights) { + for (int ix=0; ix