Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[SYSTEMML-1325] Fixes formatting issues and warnings. #838

Closed
wants to merge 1 commit into from
Closed

[SYSTEMML-1325] Fixes formatting issues and warnings. #838

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
104 changes: 52 additions & 52 deletions src/main/java/org/apache/sysml/api/DMLScript.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -376,56 +376,56 @@ else if (debug.equalsIgnoreCase("trace"))
// (core compilation and execute)
////////

/**
* The running body of DMLScript execution. This method should be called after execution properties have been correctly set,
* and customized parameters have been put into _argVals
*
* @param dmlScriptStr DML script string
* @param fnameOptConfig configuration file
* @param argVals map of argument values
* @param allArgs arguments
* @param scriptType type of script (DML or PyDML)
* @throws IOException if IOException occurs
*/
private static void execute(String dmlScriptStr, String fnameOptConfig, Map<String,String> argVals, String[] allArgs, ScriptType scriptType)
throws IOException
{
SCRIPT_TYPE = scriptType;

//print basic time and environment info
printStartExecInfo( dmlScriptStr );

//Step 1: parse configuration files & write any configuration specific global variables
DMLConfig dmlconf = DMLConfig.readConfigurationFile(fnameOptConfig);
ConfigurationManager.setGlobalConfig(dmlconf);
CompilerConfig cconf = OptimizerUtils.constructCompilerConfig(dmlconf);
ConfigurationManager.setGlobalConfig(cconf);
LOG.debug("\nDML config: \n" + dmlconf.getConfigInfo());

setGlobalFlags(dmlconf);
Program rtprog = ScriptExecutorUtils.compileRuntimeProgram(dmlScriptStr, argVals, allArgs,
scriptType, dmlconf, SystemMLAPI.DMLScript);
List<GPUContext> gCtxs = ConfigurationManager.getDMLOptions().gpu ? GPUContextPool.getAllGPUContexts() : null;

//double costs = CostEstimationWrapper.getTimeEstimate(rtprog, ExecutionContextFactory.createContext());
//System.out.println("Estimated costs: "+costs);

//Step 10: execute runtime program
ExecutionContext ec = null;
try {
ec = ScriptExecutorUtils.executeRuntimeProgram(
rtprog, dmlconf, ConfigurationManager.isStatistics() ?
ConfigurationManager.getDMLOptions().getStatisticsMaxHeavyHitters() : 0,
new LocalVariableMap(), null, SystemMLAPI.DMLScript, gCtxs);
}
finally {
if(ec != null && ec instanceof SparkExecutionContext)
((SparkExecutionContext) ec).close();
LOG.info("END DML run " + getDateTime() );
//cleanup scratch_space and all working dirs
cleanupHadoopExecution( dmlconf );
}
}
/**
* The running body of DMLScript execution. This method should be called after execution properties have been correctly set,
* and customized parameters have been put into _argVals
*
* @param dmlScriptStr DML script string
* @param fnameOptConfig configuration file
* @param argVals map of argument values
* @param allArgs arguments
* @param scriptType type of script (DML or PyDML)
* @throws IOException if IOException occurs
*/
private static void execute(String dmlScriptStr, String fnameOptConfig, Map<String,String> argVals, String[] allArgs, ScriptType scriptType)
throws IOException
{
SCRIPT_TYPE = scriptType;

//print basic time and environment info
printStartExecInfo( dmlScriptStr );

//Step 1: parse configuration files & write any configuration specific global variables
DMLConfig dmlconf = DMLConfig.readConfigurationFile(fnameOptConfig);
ConfigurationManager.setGlobalConfig(dmlconf);
CompilerConfig cconf = OptimizerUtils.constructCompilerConfig(dmlconf);
ConfigurationManager.setGlobalConfig(cconf);
LOG.debug("\nDML config: \n" + dmlconf.getConfigInfo());

setGlobalFlags(dmlconf);
Program rtprog = ScriptExecutorUtils.compileRuntimeProgram(dmlScriptStr, argVals, allArgs,
scriptType, dmlconf, SystemMLAPI.DMLScript);
List<GPUContext> gCtxs = ConfigurationManager.getDMLOptions().gpu ? GPUContextPool.getAllGPUContexts() : null;

//double costs = CostEstimationWrapper.getTimeEstimate(rtprog, ExecutionContextFactory.createContext());
//System.out.println("Estimated costs: "+costs);

//Step 10: execute runtime program
ExecutionContext ec = null;
try {
ec = ScriptExecutorUtils.executeRuntimeProgram(
rtprog, ConfigurationManager.isStatistics() ?
ConfigurationManager.getDMLOptions().getStatisticsMaxHeavyHitters() : 0,
new LocalVariableMap(), null, SystemMLAPI.DMLScript, gCtxs);
}
finally {
if(ec != null && ec instanceof SparkExecutionContext)
((SparkExecutionContext) ec).close();
LOG.info("END DML run " + getDateTime() );
//cleanup scratch_space and all working dirs
cleanupHadoopExecution( dmlconf );
}
}

/**
* Sets the global flags in DMLScript based on user provided configuration
Expand Down Expand Up @@ -547,8 +547,8 @@ private static void checkSecuritySetup(DMLConfig config)

//determine security states
boolean flagDiffUser = !( taskController.equals("org.apache.hadoop.mapred.LinuxTaskController") //runs map/reduce tasks as the current user
|| localMode // run in the same JVM anyway
|| groupNames.contains( ttGroupName) ); //user in task tracker group
|| localMode // run in the same JVM anyway
|| groupNames.contains( ttGroupName) ); //user in task tracker group
boolean flagLocalFS = fsURI==null || fsURI.getScheme().equals("file");
boolean flagSecurity = perm.equals("yes");

Expand Down
20 changes: 8 additions & 12 deletions src/main/java/org/apache/sysml/api/ScriptExecutorUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,7 +116,7 @@ public static Program compileRuntimeProgram(String script, Map<String,String> ns
boolean init) {
DMLScript.SCRIPT_TYPE = scriptType;

Program rtprog = null;
Program rtprog;

if (ConfigurationManager.isGPU() && !IS_JCUDA_AVAILABLE)
throw new RuntimeException("Incorrect usage: Cannot use the GPU backend without JCuda libraries. Hint: Include systemml-*-extra.jar (compiled using mvn package -P distribution) into the classpath.");
Expand Down Expand Up @@ -161,7 +161,7 @@ else if (!ConfigurationManager.isGPU() && ConfigurationManager.isForcedGPU())

//init working directories (before usage by following compilation steps)
if(api != SystemMLAPI.JMLC)
if ((api == SystemMLAPI.MLContext && init) || api != SystemMLAPI.MLContext)
if (api != SystemMLAPI.MLContext || init)
DMLScript.initHadoopExecution( dmlconf );


Expand Down Expand Up @@ -222,8 +222,9 @@ else if(api == SystemMLAPI.MLContext) {
ExplainCounts counts = Explain.countDistributedOperations(rtprog);
Statistics.resetNoOfCompiledJobs( counts.numJobs );
//explain plan of program (hops or runtime)
if( DMLScript.EXPLAIN != ExplainType.NONE )
System.out.println(Explain.display(prog, rtprog, DMLScript.EXPLAIN, counts));
if( ConfigurationManager.getDMLOptions().explainType != ExplainType.NONE )
System.out.println(
Explain.display(prog, rtprog, ConfigurationManager.getDMLOptions().explainType, counts));

Statistics.stopCompileTimer();
}
Expand All @@ -232,9 +233,6 @@ else if(api == SystemMLAPI.MLContext) {
// don't chain ParseException (for cleaner error output)
throw pe;
}
catch(IOException ex) {
throw new DMLException(ex);
}
catch(Exception ex) {
throw new DMLException(ex);
}
Expand All @@ -248,8 +246,6 @@ else if(api == SystemMLAPI.MLContext) {
*
* @param rtprog
* runtime program
* @param dmlconf
* dml configuration
* @param statisticsMaxHeavyHitters
* maximum number of statistics to print
* @param symbolTable
Expand All @@ -262,7 +258,7 @@ else if(api == SystemMLAPI.MLContext) {
* list of GPU contexts
* @return execution context
*/
public static ExecutionContext executeRuntimeProgram(Program rtprog, DMLConfig dmlconf, int statisticsMaxHeavyHitters,
public static ExecutionContext executeRuntimeProgram(Program rtprog, int statisticsMaxHeavyHitters,
LocalVariableMap symbolTable, HashSet<String> outputVariables,
SystemMLAPI api, List<GPUContext> gCtxs) {
boolean exceptionThrown = false;
Expand Down Expand Up @@ -299,7 +295,7 @@ public static ExecutionContext executeRuntimeProgram(Program rtprog, DMLConfig d
if(outputVariables != null) {
for(String outVar : outputVariables) {
Data data = ec.getVariable(outVar);
if(data != null && data instanceof MatrixObject) {
if(data instanceof MatrixObject) {
for(GPUContext gCtx : ec.getGPUContexts()) {
GPUObject gpuObj = ((MatrixObject)data).getGPUObject(gCtx);
if(gpuObj != null && gpuObj.isDirty()) {
Expand Down Expand Up @@ -341,4 +337,4 @@ public static ExecutionContext executeRuntimeProgram(Program rtprog, DMLConfig d
return ec;
}

}
}
8 changes: 4 additions & 4 deletions src/main/java/org/apache/sysml/api/jmlc/PreparedScript.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@
import org.apache.sysml.api.DMLException;
import org.apache.sysml.api.ScriptExecutorUtils;
import org.apache.sysml.api.ScriptExecutorUtils.SystemMLAPI;
import org.apache.sysml.api.DMLScript;
import org.apache.sysml.conf.CompilerConfig;
import org.apache.sysml.conf.ConfigurationManager;
import org.apache.sysml.conf.DMLConfig;
Expand All @@ -47,8 +46,6 @@
import org.apache.sysml.runtime.controlprogram.Program;
import org.apache.sysml.runtime.controlprogram.caching.FrameObject;
import org.apache.sysml.runtime.controlprogram.caching.MatrixObject;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContext;
import org.apache.sysml.runtime.controlprogram.context.ExecutionContextFactory;
import org.apache.sysml.runtime.instructions.cp.BooleanObject;
import org.apache.sysml.runtime.instructions.cp.Data;
import org.apache.sysml.runtime.instructions.cp.DoubleObject;
Expand Down Expand Up @@ -100,6 +97,9 @@ private PreparedScript(PreparedScript that) {
_inVarReuse = new HashMap<>(that._inVarReuse);
_dmlconf = that._dmlconf;
_cconf = that._cconf;
_isStatisticsEnabled = that._isStatisticsEnabled;
_gatherMemStats = that._gatherMemStats;
_gpuCtx = that._gpuCtx;
}

/**
Expand Down Expand Up @@ -463,7 +463,7 @@ public ResultVariables executeScript() {

//create and populate execution context
ScriptExecutorUtils.executeRuntimeProgram(
_prog, _dmlconf, ConfigurationManager.isStatistics() ?
_prog, ConfigurationManager.isStatistics() ?
ConfigurationManager.getDMLOptions().getStatisticsMaxHeavyHitters() : 0,
_vars, _outVarnames, SystemMLAPI.JMLC, _gpuCtx);

Expand Down
Loading