Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Adding an option that allows to execute any command in a condor job #6

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Adding an option that allows to execute any command in a condor job #6

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
102 changes: 23 additions & 79 deletions condor/restManagerToCondor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
sectionName = ""
jobName = ""
fileList = ""
singleCommand = ""

onlyScripts=0

Expand Down Expand Up @@ -54,7 +55,10 @@ def ProcessFilesInList( file_list ):
command = os.environ['REST_PATH'] + "/bin/restManager --c " + os.environ['PWD'] + "/" + cfgFile + " --f " + fileToProcess
if sectionName != "":
command = command + " --n " + sectionName
f.write( command + "\n" )
if( singleCommand != "" ):
f.write( singleCommand + "\n" )
else:
f.write( command + "\n" )
f.close()

st = os.stat( scriptName + ".sh" )
Expand Down Expand Up @@ -88,81 +92,6 @@ def ProcessFilesInList( file_list ):
print "---> Produced condor script : " + str( scriptName ) + "_" + str(cont) + "_" + str( jobName ) + ".condor"
print "---> To launch : " + command

if (fileList == "" ):
scriptName = "condor/" + jobName

f = open( scriptName + ".sh", "w" )
f.write("#!/bin/bash\n")
#f.write("export REST_DATAPATH="+ os.environ['REST_DATAPATH']+"\n")

for key in os.environ.keys():
if key.find( "HOME") == 0:
f.write( "export " + key + "=" + os.environ[key] +"\n" )
print( "export " + key + "=" + os.environ[key] +"\n" )
if key.find( "DATA") == 0:
f.write( "export " + key + "=" + os.environ[key] +"\n" )
print( "export " + key + "=" + os.environ[key] +"\n" )
if key.find("GDML") == 0:
f.write( "export " + key + "=" + os.environ[key] +"\n" )
print( "export " + key + "=" + os.environ[key] +"\n" )
if key.find("GEOMETRY") >= 0:
f.write( "export " + key + "=" + os.environ[key] +"\n" )
print( "export " + key + "=" + os.environ[key] +"\n" )
if key.find("REST") == 0:
f.write( "export " + key + "=" + os.environ[key] +"\n" )
print( "export " + key + "=" + os.environ[key] +"\n" )
if key.find("G4") == 0:
f.write( "export " + key + "=" + os.environ[key] +"\n" )
if key.find("PATH") == 0:
print( "export " + key + "=" + os.environ[key] +"\n" )
f.write( "export " + key + "=" + os.environ[key] +"\n" )
if key.find("LD_LIBRARY_PATH") == 0:
print( "export " + key + "=" + os.environ[key] +"\n" )
f.write( "export " + key + "=" + os.environ[key] +"\n" )
if key.find("GARFIELD_") == 0:
print( "export " + key + "=" + os.environ[key] +"\n" )
f.write( "export " + key + "=" + os.environ[key] +"\n" )
if key.find("HEED_") == 0:
f.write( "export " + key + "=" + os.environ[key] +"\n" )
if key.find("PWD") == 0:
f.write( "export " + key + "=" + os.environ[key] +"\n" )

f.write("export USER="+ os.environ['USER']+"\n\n")

command = "restManager --c " + cfgFile
if sectionName != "":
command = command + " --n " + sectionName
f.write( command + "\n" )
f.close()

st = os.stat( scriptName + ".sh" )
os.chmod( scriptName + ".sh", st.st_mode | stat.S_IEXEC)

cont = cont + 1

g = open( scriptName + ".condor", "w" )
g.write("Executable = " + scriptName + ".sh\n" )
g.write("Arguments = \n" )
g.write("Log = " + scriptName + ".log\n" )
g.write("Output = " + scriptName + ".out\n" )
g.write("Error = " + scriptName + ".err\n" )
g.write("Queue\n" )
g.close()

if onlyScripts == 0:
print "---> Launching : " + command

condorCommand = "condor_submit " + scriptName + ".condor"
print "Condor command : " + condorCommand

print "Waiting " + str(sleep) + " seconds to launch next job"
time.sleep(sleep)

print commands.getstatusoutput( condorCommand )
else:
print "---> Produced condor script : " + str( scriptName ) + "_" + str( jobName ) + ".condor"
print "---> To launch : " + command

def ProcessWithoutFile( ):
rpt = repeat
cont = 0
Expand Down Expand Up @@ -247,6 +176,13 @@ def ProcessWithoutFile( ):
print " - FILELIST. A file containing one file per line"
print " - GLOBPATTERN. A glob filename pattern using wild cards *?"
print ""
print ""
print " An additional option will allow you to launch any command (without "
print " need to call restManager) using:"
print ""
print " Usage : restManagerToCondor.py --command \"cmd + arguments\""
print ""
print ""
print " In all cases full path file location should be given!"
print ""
print " - Other options : "
Expand Down Expand Up @@ -303,17 +239,23 @@ def ProcessWithoutFile( ):
if ( sys.argv[x+1] == "--fileList" or sys.argv[x+1] == "-f" ):
fileList = sys.argv[x+2]

if ( sys.argv[x+1] == "--command" or sys.argv[x+1] == "-C" ):
singleCommand = sys.argv[x+2]

if ( sys.argv[x+1] == "--onlyScripts" or sys.argv[x+1] == "-o" ):
onlyScripts = 1


if jobName == "":
if jobName == "" and cfgFile != "":
jobName = cfgFile[cfgFile.rfind("/")+1:cfgFile.rfind(".rml")]
if jobName == "":
jobName = "noName"


if not os.path.exists("condor/" + jobName):
os.makedirs("condor/" + jobName)

if cfgFile == "":
if singleCommand == "" and cfgFile == "":
print "Please specify a RML config file list using -c flag."
sys.exit( 1 )

Expand All @@ -332,8 +274,10 @@ def ProcessWithoutFile( ):
print ( "Files to process:" )
print ( lines )

ProcessFilesInList( lines )
elif fileList == "" and singleCommand != "":
lines.append( singleCommand )
ProcessFilesInList( lines )
else:
print( " ======> Launching processing without input file <======" )
ProcessWithoutFile( )