From 018ecbe864e261a2c823985322aef68aa43765d0 Mon Sep 17 00:00:00 2001 From: Michael Grauer Date: Thu, 15 Sep 2011 18:25:14 -0400 Subject: [PATCH] BUG: Refs #212. Work towards exporting batchmake scripts. This commit breaks out much functionality into KWBatchmakeComponent.php and also further into a library (KWUtils.php). Included are tests for both of these, along with testing data. The functionality provided would allow for creating a tmp work directory in the batchmake module, copying in the needed .bms and .bmm files, compiling the batchmake script, creating the condor DAG and scripts from the .bms file, and submitting the DAG to condor. --- library/KWUtils.php | 279 +++++++++ modules/batchmake/Notification.php | 26 +- modules/batchmake/constant/module.php | 12 +- .../controllers/ConfigController.php | 4 +- .../components/KWBatchmakeComponent.php | 376 ++++++++++--- modules/batchmake/database/mysql/0.1.0.sql | 5 + modules/batchmake/models/AppDao.php | 1 + modules/batchmake/models/AppModel.php | 1 + .../batchmake/models/base/TaskModelBase.php | 53 ++ modules/batchmake/models/dao/TaskDao.php | 19 + modules/batchmake/models/pdo/TaskModel.php | 19 + .../public/css/config/config.index.css | 4 + modules/batchmake/public/images/cmake.png | Bin 0 -> 703 bytes modules/batchmake/tests/configs/.gitignore | 4 + .../tests/controllers/CMakeLists.txt | 3 + .../controllers/ConfigControllerTest.php | 528 ++++++++++++++++++ .../controllers/components/CMakeLists.txt | 6 + .../components/KWBatchmakeComponentTest.php | 407 ++++++++++++++ .../midas3batchmake/bin/AnotherApp.bmm | 0 .../testfiles/midas3batchmake/bin/MyApp2.bmm | 0 .../midas3batchmake/bin/PixelCounter.bmm | 58 ++ .../midas3batchmake/bin/TestApp1.bmm | 0 .../midas3batchmake/bin/TestApp2.bmm | 0 .../testfiles/midas3batchmake/bin/myapp.bmm | 0 .../data/testing/sphere_10.mha | Bin 0 -> 816 bytes .../data/testing/sphere_15.mha | Bin 0 -> 1250 bytes .../midas3batchmake/data/testing/sphere_5.mha | Bin 0 -> 527 bytes .../midas3batchmake/script/CompileErrors.bms | 7 + .../midas3batchmake/script/Compiles.bms | 22 + .../midas3batchmake/script/Myscript2.bms | 25 + .../midas3batchmake/script/PixelCounter.bms | 24 + .../midas3batchmake/script/anotherscript.bms | 28 + .../script/anotherscriptwitherrors.bms | 25 + .../midas3batchmake/script/bmmswitherrors.bms | 23 + .../midas3batchmake/script/cycle1.bms | 1 + .../midas3batchmake/script/cycle31.bms | 2 + .../midas3batchmake/script/cycle32.bms | 1 + .../midas3batchmake/script/cycle33.bms | 1 + .../midas3batchmake/script/myscript.bms | 26 + .../midas3batchmake/script/nocycle1.bms | 2 + .../midas3batchmake/script/nocycle2.bms | 1 + .../midas3batchmake/script/nocycle3.bms | 1 + .../midas3batchmake/script/noscripts.bms | 23 + modules/batchmake/views/config/index.phtml | 1 - tests/CMakeLists.txt | 1 + tests/library/CMakeLists.txt | 6 + tests/library/KWUtilsTest.php | 153 +++++ 47 files changed, 2092 insertions(+), 86 deletions(-) create mode 100644 library/KWUtils.php create mode 100644 modules/batchmake/database/mysql/0.1.0.sql create mode 100644 modules/batchmake/models/base/TaskModelBase.php create mode 100644 modules/batchmake/models/dao/TaskDao.php create mode 100644 modules/batchmake/models/pdo/TaskModel.php create mode 100644 modules/batchmake/public/images/cmake.png create mode 100644 modules/batchmake/tests/configs/.gitignore create mode 100644 modules/batchmake/tests/controllers/components/CMakeLists.txt create mode 100644 modules/batchmake/tests/controllers/components/KWBatchmakeComponentTest.php create mode 100644 modules/batchmake/tests/testfiles/midas3batchmake/bin/AnotherApp.bmm create mode 100644 modules/batchmake/tests/testfiles/midas3batchmake/bin/MyApp2.bmm create mode 100644 modules/batchmake/tests/testfiles/midas3batchmake/bin/PixelCounter.bmm create mode 100644 modules/batchmake/tests/testfiles/midas3batchmake/bin/TestApp1.bmm create mode 100644 modules/batchmake/tests/testfiles/midas3batchmake/bin/TestApp2.bmm create mode 100644 modules/batchmake/tests/testfiles/midas3batchmake/bin/myapp.bmm create mode 100644 modules/batchmake/tests/testfiles/midas3batchmake/data/testing/sphere_10.mha create mode 100644 modules/batchmake/tests/testfiles/midas3batchmake/data/testing/sphere_15.mha create mode 100644 modules/batchmake/tests/testfiles/midas3batchmake/data/testing/sphere_5.mha create mode 100644 modules/batchmake/tests/testfiles/midas3batchmake/script/CompileErrors.bms create mode 100644 modules/batchmake/tests/testfiles/midas3batchmake/script/Compiles.bms create mode 100644 modules/batchmake/tests/testfiles/midas3batchmake/script/Myscript2.bms create mode 100644 modules/batchmake/tests/testfiles/midas3batchmake/script/PixelCounter.bms create mode 100644 modules/batchmake/tests/testfiles/midas3batchmake/script/anotherscript.bms create mode 100644 modules/batchmake/tests/testfiles/midas3batchmake/script/anotherscriptwitherrors.bms create mode 100644 modules/batchmake/tests/testfiles/midas3batchmake/script/bmmswitherrors.bms create mode 100644 modules/batchmake/tests/testfiles/midas3batchmake/script/cycle1.bms create mode 100644 modules/batchmake/tests/testfiles/midas3batchmake/script/cycle31.bms create mode 100644 modules/batchmake/tests/testfiles/midas3batchmake/script/cycle32.bms create mode 100644 modules/batchmake/tests/testfiles/midas3batchmake/script/cycle33.bms create mode 100644 modules/batchmake/tests/testfiles/midas3batchmake/script/myscript.bms create mode 100644 modules/batchmake/tests/testfiles/midas3batchmake/script/nocycle1.bms create mode 100644 modules/batchmake/tests/testfiles/midas3batchmake/script/nocycle2.bms create mode 100644 modules/batchmake/tests/testfiles/midas3batchmake/script/nocycle3.bms create mode 100644 modules/batchmake/tests/testfiles/midas3batchmake/script/noscripts.bms create mode 100644 tests/library/CMakeLists.txt create mode 100644 tests/library/KWUtilsTest.php diff --git a/library/KWUtils.php b/library/KWUtils.php new file mode 100644 index 000000000..f1e0158ff --- /dev/null +++ b/library/KWUtils.php @@ -0,0 +1,279 @@ + +&1" : ""; + exec(KWUtils::escapeCommand($command) . $redirect_error, $output, $return_val); + } + + + /** + * @method isLinux() + * @return True if the current platform is Linux + */ + public static function isLinux() + { + return (strtolower(substr(PHP_OS, 0, 5)) == "linux"); + } + + + + /** + * @method prepareExecCommand + * will prepare an executable application and params for command line + * execution, including escaping and quoting arguments. + * @param $app_name, the application to be executed + * @param $params, an array of arguments to the application + * @return the full command line command, escaped and quoted, will throw a + * Zend_Exception if the app is not in the path and not executable + */ + public static function prepareExecCommand($app_name, $params = array()) + { + // Check if application is executable, if not, see if you can find it + // in the path + if(!KWUtils::isExecutable($app_name, false)) + { + $app_name = KWUtils::findApp($app_name, true); + } + + // escape parameters + $escapedParams = array(); + foreach($params as $param) + { + $escapedParams[] = escapeshellarg($param); + } + + // glue together app_name and params using spaces + return escapeshellarg($app_name)." ".implode(" ", $escapedParams); + } + + + /** + * @method isExecutable will return true if the app can be found and is + * executable, can optionally look in the path. + * @param string $app_name, the app to check + * @param boolean $check_in_path, if true, will search in path for app + * @return True if app_name is found and executable, False otherwise + */ + public static function isExecutable($app_name, $check_in_path = false) + { + if(!is_executable($app_name )) + { + if($check_in_path) + { + try + { + if(KWUtils::findApp($app_name, true)) + { + return true; + } + } + catch(Zend_Exception $ze) + { + return false; + } + } + return false; + } + return true; + } + + /** + * @method findApp will return the absolute path of an application + * @param $app_name, the name of the application + * @param $check_execution_flag, whether to include in the check that the + * application is executable + * @return the path to the application, throws a Zend_Exception if the app + * can't be found, or if $check_execution_flag is set and the app is not + * executable. + */ + public static function findApp($app_name, $check_execution_flag ) + { + $PHP_PATH_SEPARATOR = ":"; + // split path + $path_list = explode($PHP_PATH_SEPARATOR, getenv("PATH")); + + // loop through paths + foreach($path_list as $path) + { + $status = false; + $path_to_app = KWUtils::appendStringIfNot($path, DIRECTORY_SEPARATOR).$app_name; + if($check_execution_flag) + { + if(is_executable($path_to_app)) + { + $status = true; + break; + } + } + else + { + if(file_exists($path_to_app)) + { + $status = true; + break; + } + } + } + if(!$status) + { + throw new Zend_Exception("Failed to locate the application: [".$app_name."] [check_execution_flag:".$check_execution_flag."]"); + } + return $path_to_app; + } + + + + + /** + * @method formatAppName + * Format the application name according to the platform. + */ + public static function formatAppName($app_name) + { + if(substr(PHP_OS, 0, 3) == "WIN") + { + $app_name = KWUtils::appendStringIfNot($app_name, ".exe"); + } + return $app_name; + } + + +} diff --git a/modules/batchmake/Notification.php b/modules/batchmake/Notification.php index dbb88dad4..2945be5a2 100644 --- a/modules/batchmake/Notification.php +++ b/modules/batchmake/Notification.php @@ -21,10 +21,16 @@ class Batchmake_Notification extends MIDAS_Notification public function init() { $this->addCallBack('CALLBACK_CORE_GET_DASHBOARD', 'getDashboard'); + $this->addCallBack('CALLBACK_CORE_GET_LEFT_LINKS', 'getLeftLink'); }//end init - - /** generate Dashboard information */ + + /** + *@method getDashboard + * will generate information about this module to display on the Dashboard + *@return array with key being a string describing if the configuration of + * the module is correct or not, and value being a 1/0 for the same info. + */ public function getDashboard() { $return = array(); @@ -38,5 +44,21 @@ public function getDashboard() } return $return; } + + + /** + *@method getLeftLink + * will generate a link for this module to be displayed in the main view. + *@return ['batchmake' => [ link to batchmake module, module icon image path]] + */ + public function getLeftLink() + { + $fc = Zend_Controller_Front::getInstance(); + $moduleWebroot = $fc->getBaseUrl() . MIDAS_BATCHMAKE_MODULE; + return array(ucfirst(MIDAS_BATCHMAKE_MODULE) => array($moduleWebroot, $moduleWebRoot . '/public/images/cmake.png')); + } + } //end class + + ?> diff --git a/modules/batchmake/constant/module.php b/modules/batchmake/constant/module.php index 273a1645e..66f1f7914 100644 --- a/modules/batchmake/constant/module.php +++ b/modules/batchmake/constant/module.php @@ -16,7 +16,7 @@ define("MIDAS_BATCHMAKE_CHECK_IF_CHMODABLE", 0x10); define("MIDAS_BATCHMAKE_CHECK_IF_CHMODABLE_RW", 0x16); // 0x10 + 0x6 -define("MIDAS_BATCHMAKE_DEFAULT_MKDIR_MODE", 0775); + // Condor executables define("MIDAS_BATCHMAKE_CONDOR_STATUS", "condor_status"); @@ -26,7 +26,10 @@ // Batchmake executable define("MIDAS_BATCHMAKE_EXE", "BatchMake"); - +// Batchmake temporary execution dir +define("MIDAS_BATCHMAKE_SSP_DIR", "SSP"); + + // Extension automatically appended to dagman // description file when 'condor_dag_submit' generates it define("MIDAS_BATCHMAKE_CONDOR_DAGMAN_EXT", ".condor.sub"); @@ -88,7 +91,10 @@ define("MIDAS_BATCHMAKE_NOT_FOUND_ON_CURRENT_SYSTEM_STRING", 'Not found on the current system'); define("MIDAS_BATCHMAKE_FILE_OR_DIRECTORY_DOESNT_EXIST_STRING", "File or directory doesn't exist:"); - +define("MIDAS_BATCHMAKE_NO_SCRIPT_SPECIFIED", "No script specified"); +define("MIDAS_BATCHMAKE_NO_SCRIPT_FOUND", "No script found at: "); +define("MIDAS_BATCHMAKE_CREATE_TMP_DIR_FAILED", "Failed to create temporary directory: "); +define("MIDAS_BATCHMAKE_SYMLINK_FAILED", "Failed to create symbolic link: "); // property keys diff --git a/modules/batchmake/controllers/ConfigController.php b/modules/batchmake/controllers/ConfigController.php index b0d8fb296..1f5f413a6 100644 --- a/modules/batchmake/controllers/ConfigController.php +++ b/modules/batchmake/controllers/ConfigController.php @@ -136,7 +136,9 @@ public function testconfigAction() } $config_status = $this->ModuleComponent->KWBatchmake->testconfig($configPropertiesParamVals); - echo JsonComponent::encode($config_status); + $jsonout = JsonComponent::encode($config_status); + echo $jsonout; +// echo JsonComponent::encode($config_status); }//end testconfigAction diff --git a/modules/batchmake/controllers/components/KWBatchmakeComponent.php b/modules/batchmake/controllers/components/KWBatchmakeComponent.php index 8adae013f..dd75832b3 100644 --- a/modules/batchmake/controllers/components/KWBatchmakeComponent.php +++ b/modules/batchmake/controllers/components/KWBatchmakeComponent.php @@ -9,14 +9,17 @@ the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the above copyright notices for more information. =========================================================================*/ - +?> + MIDAS_BATCHMAKE_CHECK_IF_CHMODABLE_RW, MIDAS_BATCHMAKE_BIN_DIR_PROPERTY => MIDAS_BATCHMAKE_CHECK_IF_READABLE, @@ -31,7 +34,13 @@ class Batchmake_KWBatchmakeComponent extends AppComponent MIDAS_BATCHMAKE_CONDOR_SUBMIT_DAG => MIDAS_BATCHMAKE_CONDOR_BIN_DIR_PROPERTY, MIDAS_BATCHMAKE_EXE => MIDAS_BATCHMAKE_BIN_DIR_PROPERTY); + protected $alternateConfig = false; + /** will allow an alterate config bundle to be passed in. */ + public function setAlternateConfig($alternate) + { + $this->alternateConfig = $alternate; + } /** @@ -59,7 +68,11 @@ public function getApplicationsPaths() */ function loadApplicationConfig() { - if(file_exists(MIDAS_BATCHMAKE_MODULE_LOCAL_CONFIG)) + if($this->alternateConfig) + { + $applicationConfig = parse_ini_file($this->alternateConfig, false); + } + elseif(file_exists(MIDAS_BATCHMAKE_MODULE_LOCAL_CONFIG)) { $applicationConfig = parse_ini_file(MIDAS_BATCHMAKE_MODULE_LOCAL_CONFIG, false); } @@ -103,7 +116,7 @@ public function checkFileFlag($file, $options = 0x0) $status .= $executable ? " / Executable" : " / NotExecutable"; $ret = $ret && $executable; } - if(!$this->IsWindows() && $exist && ($options & MIDAS_BATCHMAKE_CHECK_IF_CHMODABLE)) + if(!KWUtils::isWindows() && $exist && ($options & MIDAS_BATCHMAKE_CHECK_IF_CHMODABLE)) { $chmodable = $this->IsChmodable($file); $status .= $chmodable ? " / Chmodable" : " / NotChmodable"; @@ -114,51 +127,20 @@ public function checkFileFlag($file, $options = 0x0) - /** - * @method isWindows() - * from KWUtils, may need to be moved. - * @return True if the current platform is windows - */ - function isWindows() - { - return (strtolower(substr(PHP_OS, 0, 3)) == "win"); - } - - /** - * @method isLinux() - * from KWUtils, may need to be moved. - * @return True if the current platform is Linux - */ - function isLinux() - { - return (strtolower(substr(PHP_OS, 0, 5)) == "linux"); - } - /** - * @method formatAppName - * from KWUtils, may need to be moved. - * Format the application name according to the platform. - */ - function formatAppName($app_name) - { - if(substr(PHP_OS, 0, 3) == "WIN") - { - $app_name = self::AppendStringIfNot($app_name, ".exe"); - } - return $app_name; - } + /** * @method isChmodable * Check if current PHP process has permission to change the mode * of $fileOrDirectory. - * from KWUtils, may need to be moved. + * @TODO from KWUtils, may need to be moved. * Note: If return true, the mode of the file will be MIDAS_BATCHMAKE_DEFAULT_MKDIR_MODE * On windows, return always True */ function isChmodable($fileOrDirectory) { - if($this->isWindows()) + if(KWUtils::isWindows()) { return true; } @@ -172,21 +154,24 @@ function isChmodable($fileOrDirectory) // Get permissions of the file // TODO On CIFS filesytem, even if the function GetFilePermissions call clearstatcache(), the value returned can be wrong - //$current_perms = self::GetFilePermissions( $fileOrDirectory ); - $current_perms = MIDAS_BATCHMAKE_DEFAULT_MKDIR_MODE; //self::Debug("File permissions: [file: $fileOrDirectory, mode: ".decoct($current_perms)."]"); + //$current_perms = self::GetFilePermissions( $fileOrDirectory ); + $current_perms = KWUtils::DEFAULT_MKDIR_MODE;//MIDAS_BATCHMAKE_DEFAULT_MKDIR_MODE; if($current_perms === false) { return false; } - // Try to re-apply them - if(!@chmod($fileOrDirectory, $current_perms)) + if(is_writable($fileOrDirectory)) { - //self::Debug("Failed to change mode: [file: $fileOrDirectory] to [mode: ".decoct($current_perms)."]"); - return false; + // Try to re-apply them + $return = chmod($fileOrDirectory, $current_perms); + } + else + { + $return = false; } - return true; + return $return; } /** @@ -234,7 +219,7 @@ public function testconfig($configPropertiesParamVals = NULL) { $configPropertiesParamVals = $this->getApplicationConfigProperties(); } - + $configPropertiesRequirements = $this->getConfigPropertiesRequirements(); foreach($configPropertiesRequirements as $configProperty => $configPropertyRequirement) { @@ -253,7 +238,7 @@ public function testconfig($configPropertiesParamVals = NULL) else { // property doesn't exist, both the property and global config are in error - $configStatus[] = array(MIDAS_BATCHMAKE_PROPERTY_KEY => $configProperty, MIDAS_BATCHMAKE_STATUS_KEY => CONFIG_VALUE_MISSING, MIDAS_BATCHMAKE_TYPE_KEY => MIDAS_BATCHMAKE_STATUS_TYPE_ERROR); + $configStatus[] = array(MIDAS_BATCHMAKE_PROPERTY_KEY => $configProperty, MIDAS_BATCHMAKE_STATUS_KEY => MIDAS_BATCHMAKE_CONFIG_VALUE_MISSING, MIDAS_BATCHMAKE_TYPE_KEY => MIDAS_BATCHMAKE_STATUS_TYPE_ERROR); $total_config_correct = 0; } } @@ -263,7 +248,7 @@ public function testconfig($configPropertiesParamVals = NULL) $appsPaths = $this->getApplicationsPaths(); foreach($appsPaths as $app => $pathProperty) { - $appPath = $configPropertiesParamVals[$pathProperty] ."/" . $this->formatAppName($app); + $appPath = $configPropertiesParamVals[$pathProperty] ."/" . KWUtils::formatAppName($app); list($result, $status) = $this->checkFileFlag($appPath, MIDAS_BATCHMAKE_CHECK_IF_EXECUTABLE); Zend_Loader::loadClass("InternationalizationComponent", BASE_PATH.'/core/controllers/components'); @@ -314,13 +299,13 @@ public function testconfig($configPropertiesParamVals = NULL) /** * @method isConfigCorrect - * helper method to return 1 if the config is correct, 0 otherwise - * @return 1 if config correct, 0 otherwise + * helper method to return true if the config is correct, false otherwise + * @return true if config correct, false otherwise */ public function isConfigCorrect() { $applicationConfig = $this->testconfig(); - return $applicationConfig[0]; + return $applicationConfig[0] == 1; } @@ -346,55 +331,288 @@ public function getBatchmakeScripts() } -/* - // Glob through the tasks directory for xml files - $tasks = array(); - foreach(KwUtils::GlobRec($this->batchmake_task_directory."/","*.xml") as $filename) + + + + + /** + * @method findAndSymLinkDependentBatchmakeScripts + * will look in the tmpDir for a batchmake script and symlink it to the + * tmpDir, will then find any batchmake scripts that need to be included + * other than a config script, and symlink them in from the scriptdir, + * and for each of these additional scripts, will perform the same + * operation (symlinking included batchmake scripts), + * will throw a Zend_Exception if any symlink fails or if a target file + * doesn't exist. + * @param $scriptDir the batchmake script dir + * @param $tmpDir the temporary work dir + * @param $scriptName the original batchmake script + * @param $processed a list of those scripts already processed + * @return the array of scripts processed + */ + public function findAndSymLinkDependentBatchmakeScriptsWithCycleDetection($scriptDir, $tmpDir, $scriptName, $processed = array(), &$currentPath = array()) + { + // check for cycles + if(array_search($scriptName, $currentPath) !== false) + { + throw new Zend_Exception("Cycle found in the include graph of batchmake scripts."); + } + // push this script onto the currentPath + $currentPath[] = $scriptName; + // don't process any already processed + if(!array_key_exists($scriptName, $processed)) { - $taskname = basename($filename, ".xml"); + // symlink the top level scrip + $scriptLink = $tmpDir . '/' . $scriptName; + $scriptTarget = $scriptDir . '/' . $scriptName; + if(!file_exists($scriptTarget) || !symlink($scriptTarget, $scriptLink)) + { + throw new Zend_Exception($scriptTarget . ' could not be sym-linked to ' . $scriptLink); + } + // now consider this script to be processed + $processed[$scriptName] = $scriptName; + } - $ret = KwBatchmakeModule::ValidateTaskname( $taskname ); - $tasks[] = array( "name" => $taskname, "error" => !$ret); + // read through the script looking for includes + $contents = file_get_contents($scriptDir . '/' . $scriptName); + // looking for lines like + // Include(PixelCounter.config.bms) + // /i means case insensitive search + $pattern = '/include\s*\(\s*(\S*)\s*\)/i'; + preg_match_all($pattern, $contents, $matches); + // ensure that there actually are matches + if($matches && count($matches) > 1) + { + // we just want the subpattern match, not the full match + // the subpattern match is the name of the included file + $subpatternMatches = $matches[1]; + // now that we have the matches, we only want the ones that are not .config.bms + foreach($subpatternMatches as $ind => $includeName) + { + // only want the includes that are not .config.bms scripts + if(strpos($includeName, '.config.bms') === false) + { + // recursively process this script, updating the $processed list upon success + // essentially performing depth first search in a graph + // there could be a problem with a cycle in the include graph, + // so pass along the currentPath + $processed = $this->findAndSymLinkDependentBatchmakeScriptsWithCycleDetection($scriptDir, $tmpDir, $includeName, $processed, $currentPath); + } + } } + // pop this script off of the current path + array_pop($currentPath); + // return the processed list + return $processed; + } + + + + + + + + + + + + + + + + - - /** Recursive glob * / - static function GlobRec($path, $match = false) + + + + + + + + + /** + * @method findAndSymLinkDependentBmms + * will look in the $tmpDir for all batchmake scripts that are passed + * in the array $bmScripts, for each of these, it will find all of the apps + * included in them using the SetApp Batchmake command, and sym link the + * corresponding bmm file to the tmpDir, these bmm files are expected to be + * in the $binDir, will throw a Zend_Exception if any symlink fails or if a + * bmm file doesn't exist, or if one of the batchmake scripts doesn't exist. + * @param $binDir the bin dir with the bmm files + * @param $tmpDir the temporary work dir + * @param $bmScripts the array of Batchmake scripts in the $tmpDir to process + * @return an array of [ bmmfile => bmScript where bmmfile first found ] + */ + public function findAndSymLinkDependentBmms($binDir, $tmpDir, $bmScripts) { - // TODO Possible improvement - See http://us2.php.net/manual/en/function.glob.php#87221 - if (!function_exists('fnmatch')) + // initialize the list of bmms that have been processed + $processed = array(); + foreach($bmScripts as $bmScript) { - if (!$match) + $scriptPath = $tmpDir . '/' . $bmScript; + if(!file_exists($scriptPath)) { - return glob($path); + throw new Zend_Exception($scriptPath . ' could not be found'); } - return glob($path.$match); + $contents = file_get_contents($scriptPath); + // /i means case insensitive search + // read through the script looking for lines like + // SetApp(pixelCounter @PixelCounter) + $pattern = '/setapp\s*\(\s*\S*\s*@(\S*)\s*\)/i'; + preg_match_all($pattern, $contents, $matches); + // ensure that there actually are matches + if($matches && count($matches) > 1) + { + // we just want the subpattern match, not the full match + // the subpattern match is the name of the included file + $subpatternMatches = $matches[1]; + // now that we have the matches, get the app names to use for the bmm + foreach($subpatternMatches as $ind => $appName) + { + if(!array_key_exists($appName, $processed)) + { + $bmmTarget = $binDir . '/' . $appName . '.bmm'; + $bmmLink = $tmpDir . '/' . $appName . '.bmm'; + if(!file_exists($bmmTarget) || !symlink($bmmTarget, $bmmLink)) + { + throw new Zend_Exception($bmmTarget . ' could not be sym-linked to ' . $bmmLink); + } + // track which bmScript we first saw this app in + $processed[$appName] = $bmScript; + } + } + } } - $dir = @opendir($path); - if ($dir == false) + return $processed; + } + + + + + + + + + + + + + + + + + + + + + /** + * @method compileBatchMakeScript will check that the passed in $batchmakescript + * in the passed in $tmpDir will compile without errors. + * @param string $appDir directory where binary applications are located + * @param string $binDir directory where the BatchMake binary is located + * @param string $tmpDir directory where the work for SSP should be done + * @param string $bmScript name of the script, should be in $tmpDir + * @return type + */ + public function compileBatchMakeScript($appDir, $binDir, $tmpDir, $bmScript) + { + // Prepare command + $params = array( + '-ap', $appDir, + '-p', $tmpDir, + '-c', $tmpDir.$bmScript, + ); + $cmd = KWUtils::prepareExecCommand($binDir . '/'. MIDAS_BATCHMAKE_EXE, $params); + if($cmd === false) + { + return false; + } + + // Run command + KWUtils::exec($cmd, $output, $tmpDir, $returnVal); + + if($returnVal !== 0) { - return array (); + throw new Zend_Exception("Failed to run: [".$cmd."], output: [".implode(",", $output )."]"); } - while ($File = readdir($dir)) + + // if BatchMake reports errors, throw an exception + foreach($output as $ind => $val) { - if ($File != "." && $File != "..") - { - if (!$match || fnmatch($match, $File)) + if(preg_match("/(\d+) error/", $val, $matches)) + { + // number of errors is index 1, this is based on BatchMake's output + // it will output the number of errors even if 0 + if($matches[1] == "0") + { + return true; + } + else { - $result[] = $path.$File; + throw new Zend_Exception("Compiling script [".$bmScript."] yielded output: [".implode(",", $output )."]"); } } } - closedir($dir); - if ( empty($result)) + + throw new Zend_Exception("Error in BatchMake script, the compile step didn't report errors, output: [".implode(",", $cmd_output )."]"); + } + + + + /** + * @method generateCondorDag will create condor scripts and a condor dag + * from the batchmake script $bmScript, in the directory $tmpDir. + * @param type $appDir + * @param type $tmpDir + * @param type $binDir + * @param type $bmScript + */ + public function generateCondorDag($appDir, $tmpDir, $binDir, $bmScript) + { + $dagName = $bmScript.'.dagjob'; + + // Prepare command + $params = array( + '-ap', $appDir, + '-p', $tmpDir, + '--condor', $tmpDir.$bmScript, $tmpDir.$dagName, + ); + + $cmd = KWUtils::prepareExecCommand($binDir . '/'. MIDAS_BATCHMAKE_EXE, $params); + + // Run command + KWUtils::exec($cmd, $output, $tmpDir, $returnVal); + + if($returnVal !== 0) { - return array (); + throw new Zend_Exception("Failed to run: [".$cmd."], output: [".implode(",", $cmd_output )."]"); + } + return $dagName; + } + + /** + * @method condorSubmitDag will + * @param type $condorBinDir + * @param type $tmpDir + * @param type $dagScript + */ + public function condorSubmitDag($condorBinDir, $tmpDir, $dagScript) + { + // Prepare command + $params = array($dagScript); + + $cmd = KWUtils::prepareExecCommand($condorBinDir . '/'. MIDAS_BATCHMAKE_CONDOR_SUBMIT_DAG, $params); + + // Run command + KWUtils::exec($cmd, $output, $tmpDir, $returnVal); + + if($returnVal !== 0) + { + throw new Zend_Exception("Failed to run: [".$cmd."], output: [".implode(",", $cmd_output )."]"); } - return $result; } - */ diff --git a/modules/batchmake/database/mysql/0.1.0.sql b/modules/batchmake/database/mysql/0.1.0.sql new file mode 100644 index 000000000..13beaf45c --- /dev/null +++ b/modules/batchmake/database/mysql/0.1.0.sql @@ -0,0 +1,5 @@ +CREATE TABLE batchmake_task ( + batchmake_task_id bigint(20) NOT NULL AUTO_INCREMENT, + user_id bigint(20) NOT NULL, + PRIMARY KEY (batchmake_task_id) +); diff --git a/modules/batchmake/models/AppDao.php b/modules/batchmake/models/AppDao.php index ae4bef064..cc4d920b8 100644 --- a/modules/batchmake/models/AppDao.php +++ b/modules/batchmake/models/AppDao.php @@ -12,6 +12,7 @@ class Batchmake_AppDao extends MIDAS_GlobalDao { + public $moduleName='batchmake'; } //end class diff --git a/modules/batchmake/models/AppModel.php b/modules/batchmake/models/AppModel.php index 80489f3aa..895a0c4a8 100644 --- a/modules/batchmake/models/AppModel.php +++ b/modules/batchmake/models/AppModel.php @@ -12,6 +12,7 @@ class Batchmake_AppModel extends MIDASModel { + public $moduleName='batchmake'; } diff --git a/modules/batchmake/models/base/TaskModelBase.php b/modules/batchmake/models/base/TaskModelBase.php new file mode 100644 index 000000000..3a1f3483a --- /dev/null +++ b/modules/batchmake/models/base/TaskModelBase.php @@ -0,0 +1,53 @@ +_name = 'batchmake_task'; + $this->_key = 'batchmake_task_id'; + + + + $this->_mainData = array( + 'batchmake_task_id' => array('type' => MIDAS_DATA), + 'user_id' => array('type' => MIDAS_DATA, ) + ); + $this->initialize(); // required + } + + + /** Create a task + * @return TaskDao */ + function createTask($userDao)//, $type, $ressource, $communityDao = null) + { + if(!$userDao instanceof UserDao)// && !is_numeric($type) && !is_object($ressource)) + { + throw new Zend_Exception("Error parameters."); + } + $this->loadDaoClass('TaskDao','batchmake'); + $task = new Batchmake_TaskDao(); + $task->setUserId($userDao->getKey()); + $this->save($task); + + return $task; + } // end createTask() + + + + + + + +} // end class Batchmake_TaskModelBase + +?> \ No newline at end of file diff --git a/modules/batchmake/models/dao/TaskDao.php b/modules/batchmake/models/dao/TaskDao.php new file mode 100644 index 000000000..77010a48c --- /dev/null +++ b/modules/batchmake/models/dao/TaskDao.php @@ -0,0 +1,19 @@ + diff --git a/modules/batchmake/models/pdo/TaskModel.php b/modules/batchmake/models/pdo/TaskModel.php new file mode 100644 index 000000000..601b9ae38 --- /dev/null +++ b/modules/batchmake/models/pdo/TaskModel.php @@ -0,0 +1,19 @@ + diff --git a/modules/batchmake/public/css/config/config.index.css b/modules/batchmake/public/css/config/config.index.css index d534a6140..7398f99b5 100644 --- a/modules/batchmake/public/css/config/config.index.css +++ b/modules/batchmake/public/css/config/config.index.css @@ -27,3 +27,7 @@ form.genericForm .info { color:green; } form.genericForm .warning { font-weight: bold; color:orange; } form.genericForm .error { font-weight: bold; color:red; } div.content-batchmake-info .desc { font-weight: bold; margin-top: 0.8em; font-size: 0.9em; display: block; } + + +#batchmake_config_status.info { color:green; } +#batchmake_config_status.error { font-weight: bold; color:red; } \ No newline at end of file diff --git a/modules/batchmake/public/images/cmake.png b/modules/batchmake/public/images/cmake.png new file mode 100644 index 0000000000000000000000000000000000000000..1b6fc58e1972a4558478ece876dda928e316c6e5 GIT binary patch literal 703 zcmV;w0zmzVP)6{tDtCQSc;ApA4BP`d1GIkTJ18rk-v zIkz$0TW7vN*^E5!j`~_V#}K{v?@{O)2G z;Ivq5qs`4Xn4DY^-JhYi*U0$zAm1Vpv<3sREaMVRu2xl1rqzz@_Chl-zg@kn>n;-$ zKZvcrr1|YLt|U`h4ZFE3~~*naaE&5DI{} zdp3kNh2lFBf3id)asIu7zAr8gXJtlwJ|xGfqxn>Sg7t71DIzvzca#xIi@fyqycH`9 ly^sM;uU3#+&hkru0RZod$S%{()2aXf002ovPDHLkV1hn0Kgj?9 literal 0 HcmV?d00001 diff --git a/modules/batchmake/tests/configs/.gitignore b/modules/batchmake/tests/configs/.gitignore new file mode 100644 index 000000000..8a0c9a5dd --- /dev/null +++ b/modules/batchmake/tests/configs/.gitignore @@ -0,0 +1,4 @@ +*.local.ini +*.local.ini.old +*.local.ini.test +termsofservice.txt \ No newline at end of file diff --git a/modules/batchmake/tests/controllers/CMakeLists.txt b/modules/batchmake/tests/controllers/CMakeLists.txt index 9391e5a86..52978f0d4 100644 --- a/modules/batchmake/tests/controllers/CMakeLists.txt +++ b/modules/batchmake/tests/controllers/CMakeLists.txt @@ -1,4 +1,7 @@ +add_subdirectory( components ) + add_midas_test( BatchmakeConfigController ConfigControllerTest.php ) # Style add_midas_style_test( StyleBatchmakeControllers ${CMAKE_SOURCE_DIR}/modules/batchmake/controllers ) +add_midas_style_test( StyleTestsBatchmakeComponents ${CMAKE_SOURCE_DIR}/modules/batchmake/tests/controllers/components ) \ No newline at end of file diff --git a/modules/batchmake/tests/controllers/ConfigControllerTest.php b/modules/batchmake/tests/controllers/ConfigControllerTest.php index f5e83fc32..271325b9f 100644 --- a/modules/batchmake/tests/controllers/ConfigControllerTest.php +++ b/modules/batchmake/tests/controllers/ConfigControllerTest.php @@ -16,13 +16,30 @@ class ConfigControllerTest extends ControllerTestCase { + protected $kwBatchmakeComponent; + protected $applicationConfig; + + /** set up tests*/ public function setUp() { $this->setupDatabase(array('default')); + $this->_models = array('User'); + //$this->_daos = array('User');// + //$this->_moduleModels = array('Task');// $this->enabledModules = array('batchmake'); + + + require_once BASE_PATH.'/modules/batchmake/controllers/components/KWBatchmakeComponent.php'; + $this->kwBatchmakeComponent = new Batchmake_KWBatchmakeComponent(); + $this->kwBatchmakeComponent->setAlternateConfig(BASE_PATH.'/modules/batchmake/tests/configs/module.local.ini'); + $this->applicationConfig = $this->kwBatchmakeComponent->getApplicationConfigProperties(); + + parent::setUp(); } + + /** test index action*/ public function testIndexAction() @@ -36,6 +53,517 @@ public function testIndexAction() { $this->fail('Unable to find body element'); } + /* + // + $usersFile = $this->loadData('User', 'default'); + var_dump($usersFile); + $userDao = $this->User->load($usersFile[0]->getKey()); + echo $userDao->getKey(); + echo 'admin='.$userDao->getAdmin(); + $userDao->setAdmin(1); + echo 'admin='.$userDao->getAdmin(); + $this->User->save($userDao); + $userDao = $this->User->load($usersFile[0]->getKey()); + echo 'admin='.$userDao->getAdmin(); + + + + + $modelLoad = new MIDAS_ModelLoader(); + $batchmakeTaskModel = $modelLoad->loadModel('Task', 'batchmake'); + $taskDao = $batchmakeTaskModel->createTask($userDao); + //$preKey = $userApiModel->getByAppAndUser('Default', $userDao)->getApikey(); + //$this->assertEquals(strlen($preKey), 32); + + + // create a task + //$taskDao = $this->Batchmake_Task->createTask($userDao); + // check for app, how? + // + // + // create a dir + // copy bms and bmm to dir + // + // + + $userId = $taskDao->getUserId(); + $taskId = $taskDao->getKey(); + echo "task:".$taskId." for user:".$userId; + */ + } + + // TEST KWBATCHMAKE COMPONENTS + // want a better way of doing this, but here is a start + + /** + * tests config setup, relies on an alternate testing config to be defined, + * these properties should all point to the batchmake module testfiles dirs. + */ + // should put the kwBatchmakeComponent as an instance variable? + // hopefully that would remove all of the redundant calls to setting it up + public function testIsConfigCorrect() + { + require_once BASE_PATH.'/modules/batchmake/controllers/components/KWBatchmakeComponent.php'; + $kwBatchmakeComponent = new Batchmake_KWBatchmakeComponent(); + $kwBatchmakeComponent->setAlternateConfig(BASE_PATH.'/modules/batchmake/tests/configs/module.local.ini'); + $this->assertTrue($kwBatchmakeComponent->isConfigCorrect()); + } + + /** + * tests that all the bmScripts that have been entered for testing are found + */ + public function testGetBatchmakeScripts() + { + require_once BASE_PATH.'/modules/batchmake/controllers/components/KWBatchmakeComponent.php'; + $kwBatchmakeComponent = new Batchmake_KWBatchmakeComponent(); + $kwBatchmakeComponent->setAlternateConfig(BASE_PATH.'/modules/batchmake/tests/configs/module.local.ini'); + $foundTestScripts = $kwBatchmakeComponent->getBatchmakeScripts(); + $expectedTestScripts = array("Myscript2.bms","PixelCounter.bms","anotherscript.bms", + "anotherscriptwitherrors.bms","bmmswitherrors.bms","myscript.bms","noscripts.bms"); + foreach($expectedTestScripts as $script) + { + $this->assertContains($script, $foundTestScripts); + } + } + + /** + * @TODO some model testing, and better ways of loading the models + * test creating a model and creating the right subdirs + */ + + public function testCreateSubDirectories() + { + // test creating directories, in the same setup as would be used for + // batchmake processing. so create a task, then create a nested + // set of directories based on the taskId + require_once BASE_PATH.'/modules/batchmake/controllers/components/KWBatchmakeComponent.php'; + $kwBatchmakeComponent = new Batchmake_KWBatchmakeComponent(); + $kwBatchmakeComponent->setAlternateConfig(BASE_PATH.'/modules/batchmake/tests/configs/module.local.ini'); + $applicationConfig = $kwBatchmakeComponent->getApplicationConfigProperties(); + $modelLoad = new MIDAS_ModelLoader(); + $batchmakeTaskModel = $modelLoad->loadModel('Task', 'batchmake'); + $usersFile = $this->loadData('User', 'default'); + $userDao = $this->User->load($usersFile[0]->getKey()); + $taskDao = $batchmakeTaskModel->createTask($userDao); + $userId = $taskDao->getUserId(); + $taskId = $taskDao->getKey(); + $subdirs = array(MIDAS_BATCHMAKE_SSP_DIR, $userId, $taskId); + $tmpDir = $kwBatchmakeComponent->createSubDirectories($applicationConfig[MIDAS_BATCHMAKE_TMP_DIR_PROPERTY] . "/", $subdirs); + // now check that all the subdirs have been created + $pathToCheck = $applicationConfig[MIDAS_BATCHMAKE_TMP_DIR_PROPERTY] . '/'; + $this->assertFileExists($pathToCheck); + foreach($subdirs as $subdir) + { + $pathToCheck = $pathToCheck . '/' . $subdir; + $this->assertFileExists($pathToCheck); + $this->assertTrue(is_dir($pathToCheck)); + } + } + + + protected function clearDirFiles($dirToClear) + { + foreach(scandir($dirToClear) as $filename) + { + if($filename && $filename != '.' && $filename != '..') + { + unlink($dirToClear.'/'.$filename); + } + } } + + protected function fASLDBSWCDtestcase($scriptDir, $tmpDir, $scriptName, $expectedSet) + { + // clear the directory of any existing files + $this->clearDirFiles($tmpDir); + + // try symlinking all the batchmake files starting with $scriptName + $bmScriptsProcessed = $this->kwBatchmakeComponent->findAndSymLinkDependentBatchmakeScriptsWithCycleDetection($scriptDir,$tmpDir,$scriptName); + + // check that the correct batchmake scripts are there, and only those + // easiest just to add '.' and '..' to expected list + $expectedSet[] = '..'; + $expectedSet[] = '.'; + sort($expectedSet); + + $foundScripts = scandir($tmpDir); + sort($foundScripts); + + $this->assertEquals($expectedSet, $foundScripts, "Expected batchmake scripts not found rooted from ".$scriptName); + + // add in '.' and '..' + $bmScriptsProcessed[] = '.'; + $bmScriptsProcessed[] = '..'; + sort($bmScriptsProcessed); + + // also check that the set of scripts returned from the method is this same set + $this->assertEquals($expectedSet, $bmScriptsProcessed, "Expected batchmake scripts not equal to those returned from processing ".$scriptName); + } + + protected function fASLDBSWCDtestcaseException($scriptDir, $tmpDir, $scriptName) + { + try + { + // need to suppress error output to keep test from failing, despite exception being caught + $bmScripts = @$this->kwBatchmakeComponent->findAndSymLinkDependentBatchmakeScriptsWithCycleDetection($scriptDir,$tmpDir,$scriptName); + $this->fail('Expected an exception for $scriptName, but did not get one.'); + } + catch (Zend_Exception $ze) + { + // this is the correct behavior + } + } + + + + public function testFindAndSymLinkDependentBatchmakeScripts() + { + require_once BASE_PATH.'/modules/batchmake/controllers/components/KWBatchmakeComponent.php'; + $kwBatchmakeComponent = new Batchmake_KWBatchmakeComponent(); + $kwBatchmakeComponent->setAlternateConfig(BASE_PATH.'/modules/batchmake/tests/configs/module.local.ini'); + $applicationConfig = $kwBatchmakeComponent->getApplicationConfigProperties(); + // create a task + $modelLoad = new MIDAS_ModelLoader(); + $batchmakeTaskModel = $modelLoad->loadModel('Task', 'batchmake'); + $usersFile = $this->loadData('User', 'default'); + $userDao = $this->User->load($usersFile[0]->getKey()); + $taskDao = $batchmakeTaskModel->createTask($userDao); + $userId = $taskDao->getUserId(); + $taskId = $taskDao->getKey(); + $subdirs = array(MIDAS_BATCHMAKE_SSP_DIR, $userId, $taskId); + // create a tmpDir based on the task and user + $tmpDir = $kwBatchmakeComponent->createSubDirectories($applicationConfig[MIDAS_BATCHMAKE_TMP_DIR_PROPERTY] . "/", $subdirs); + $scriptDir = $applicationConfig[MIDAS_BATCHMAKE_SCRIPT_DIR_PROPERTY]; + + + $scriptName = 'anotherscript.bms'; + $expectedSet = array("myscript.bms","Myscript2.bms", + "anotherscript.bms","noscripts.bms","PixelCounter.bms"); + $this->fASLDBSWCDtestcase($scriptDir, $tmpDir, $scriptName, $expectedSet); + + $scriptName = "noscripts.bms"; + $expectedSet = array("noscripts.bms"); + $this->fASLDBSWCDtestcase($scriptDir, $tmpDir, $scriptName, $expectedSet); + + // try symlinking all the batchmake files starting with anotherscriptwitherrors.bms + // expect an exception, as this script includes a non-existent script + $scriptName = 'anotherscriptwitherrors.bms'; + $this->fASLDBSWCDtestcaseException($scriptDir, $tmpDir, $scriptName); + + // cycle detection tests + + // check a script with no cycle,1->2, 1->3, 3->2 + // clear the directory of the symlinked files + $scriptName = "nocycle1.bms"; + $expectedSet = array("nocycle1.bms","nocycle2.bms","nocycle3.bms"); + $this->fASLDBSWCDtestcase($scriptDir, $tmpDir, $scriptName, $expectedSet); + + // expect an exception, as this script has a simple cycle + // 1->1 + $scriptName = 'cycle1.bms'; + $this->fASLDBSWCDtestcaseException($scriptDir, $tmpDir, $scriptName); + + // check a script with a more complex cycle, 1->2, 1->3, 2->3, 3->2 + $scriptName = 'cycle31.bms'; + $this->fASLDBSWCDtestcaseException($scriptDir, $tmpDir, $scriptName); + } + + + public function xtestFindAndSymLinkDependentBmms() + { + require_once BASE_PATH.'/modules/batchmake/controllers/components/KWBatchmakeComponent.php'; + $kwBatchmakeComponent = new Batchmake_KWBatchmakeComponent(); + $kwBatchmakeComponent->setAlternateConfig(BASE_PATH.'/modules/batchmake/tests/configs/module.local.ini'); + $applicationConfig = $kwBatchmakeComponent->getApplicationConfigProperties(); + // create a task + $modelLoad = new MIDAS_ModelLoader(); + $batchmakeTaskModel = $modelLoad->loadModel('Task', 'batchmake'); + $usersFile = $this->loadData('User', 'default'); + $userDao = $this->User->load($usersFile[0]->getKey()); + $taskDao = $batchmakeTaskModel->createTask($userDao); + $userId = $taskDao->getUserId(); + $taskId = $taskDao->getKey(); + $subdirs = array(MIDAS_BATCHMAKE_SSP_DIR, $userId, $taskId); + // create a tmpDir based on the task and user + $tmpDir = $kwBatchmakeComponent->createSubDirectories($applicationConfig[MIDAS_BATCHMAKE_TMP_DIR_PROPERTY] . "/", $subdirs); + + $scriptDir = $applicationConfig[MIDAS_BATCHMAKE_SCRIPT_DIR_PROPERTY]; + $appDir = $applicationConfig[MIDAS_BATCHMAKE_APP_DIR_PROPERTY]; + + // now try symlinking all the batchmake files starting with anotherscript.bms + $scriptName = 'anotherscript.bms'; + $bmScripts_anotherscript = $kwBatchmakeComponent->findAndSymLinkDependentBatchmakeScripts($scriptDir,$tmpDir,$scriptName); + + + $bmms = $kwBatchmakeComponent->findAndSymLinkDependentBmms($appDir, $tmpDir, $bmScripts_anotherscript); + // these come as [ name of app => script where found ] + // convert them to a form useful for comparison + $processedBmms_anotherscript = array(); + foreach($bmms as $bmm=>$script) + { + $processedBmms_anotherscript[] = $bmm.'.bmm'; + } + sort($processedBmms_anotherscript); + + $globOutput = glob($tmpDir.'/*.bmm'); + // strip off the path + $foundBmms_anotherscript = array(); + foreach($globOutput as $bmm) + { + $foundBmms_anotherscript[] = basename($bmm); + } + sort( $foundBmms_anotherscript); + + $expectedBmms_anotherscript = array("AnotherApp.bmm","MyApp2.bmm", + "PixelCounter.bmm","TestApp1.bmm","TestApp2.bmm","myapp.bmm"); + sort($expectedBmms_anotherscript); + + // compare the three arrays + $this->assertEquals($processedBmms_anotherscript, $expectedBmms_anotherscript, "BMMs: processed != expected, for anotherscript.bms"); + $this->assertEquals($processedBmms_anotherscript, $foundBmms_anotherscript, "BMMs: processed != found, for anotherscript.bms"); + + } + + + + + + + + public function xtestExec() + { + // not sure how to test this exactly, for now create a tmp dir, check + // the value of pwd in it + require_once BASE_PATH.'/modules/batchmake/controllers/components/KWBatchmakeComponent.php'; + $kwBatchmakeComponent = new Batchmake_KWBatchmakeComponent(); + $kwBatchmakeComponent->setAlternateConfig(BASE_PATH.'/modules/batchmake/tests/configs/module.local.ini'); + $applicationConfig = $kwBatchmakeComponent->getApplicationConfigProperties(); + $tmpDir = $applicationConfig[MIDAS_BATCHMAKE_TMP_DIR_PROPERTY]; + // delete any old files used by this test + $execDir = $tmpDir . '/' . 'exectmp'; + if(is_dir($execDir)) + { + rmdir($execDir); + } + mkdir($execDir); + $cmd = 'pwd'; + $chdir = $execDir; + $kwBatchmakeComponent->exec($cmd, $output, $chdir, $returnVal); + // $output should have one value, the same as execDir + $this->assertEquals($execDir, $output[0]); + // return_val should be 0 + $this->assertEquals($returnVal,0); + } + + public function xtestAppendStringIfNot() + { + // try one that doesn't have the suffix: + $subject = 'blah'; + $ext = '.exe'; + $subject = $this->kwBatchmakeComponent->appendStringIfNot($subject, $ext); + $this->assertEquals($subject,'blah.exe'); + // now try one that already has the suffix + $subject = 'blah'; + $ext = '.exe'; + $subject = $this->kwBatchmakeComponent->appendStringIfNot($subject, $ext); + $this->assertEquals($subject,'blah.exe'); + } + + public function xtestFindApp() + { + // first try something that should be in the path, php, and check that it + // is executable + $pathToApp = $this->kwBatchmakeComponent->findApp('php', true); + // now try something that is unlikely to be in the path + try + { + $pathToApp = $this->kwBatchmakeComponent->findApp('php_exe_that_is_vanishingly_likley_to_be_in_the_path', true); + $this->fail('Should have caught exception but did not, testFindApp'); + } + catch(Zend_Exception $ze) + { + // correct behavior + } + } + + public function xtestIsExecutable() + { + // this is tricky to test, as it is hard to make assumptions that hold + // up across platforms + // + // for now assume that 'pwd' will not be found + $this->assertFalse($this->kwBatchmakeComponent->isExecutable('pwd',false)); + // but 'pwd' will be found in the path + $this->assertTrue($this->kwBatchmakeComponent->isExecutable('pwd',true)); + } + + + + + public function xtestPrepareExecCommand() + { + $returnVal = $this->kwBatchmakeComponent->prepareExecCommand('php', array('blah1','blah2','blah3')); + $appPath = $this->kwBatchmakeComponent->findApp('php',true); + $this->assertEquals($returnVal,"'".$appPath."' 'blah1' 'blah2' 'blah3'"); + } + + public function xtestCompileBatchMakeScript() + { + // create a task + $modelLoad = new MIDAS_ModelLoader(); + $batchmakeTaskModel = $modelLoad->loadModel('Task', 'batchmake'); + $usersFile = $this->loadData('User', 'default'); + $userDao = $this->User->load($usersFile[0]->getKey()); + $taskDao = $batchmakeTaskModel->createTask($userDao); + $userId = $taskDao->getUserId(); + $taskId = $taskDao->getKey(); + $subdirs = array(MIDAS_BATCHMAKE_SSP_DIR, $userId, $taskId); + // create a tmpDir based on the task and user + $applicationConfig = $this->kwBatchmakeComponent->getApplicationConfigProperties(); + $tmpDir = $this->kwBatchmakeComponent->createSubDirectories($applicationConfig[MIDAS_BATCHMAKE_TMP_DIR_PROPERTY] . "/", $subdirs); + + $scriptDir = $applicationConfig[MIDAS_BATCHMAKE_SCRIPT_DIR_PROPERTY]; + $binDir = $applicationConfig[MIDAS_BATCHMAKE_BIN_DIR_PROPERTY]; + $appDir = $applicationConfig[MIDAS_BATCHMAKE_APP_DIR_PROPERTY]; + + // a script that compiles + $scriptName = 'Compiles.bms'; + $bmScripts = $this->kwBatchmakeComponent->findAndSymLinkDependentBatchmakeScripts($scriptDir,$tmpDir,$scriptName); + $bmms = $this->kwBatchmakeComponent->findAndSymLinkDependentBmms($appDir, $tmpDir, $bmScripts); + + // first try with a bad path to BatchMake + $badBinDir = '/a/dir/not/likely/to/exist'; + try + { + $this->kwBatchmakeComponent->compileBatchMakeScript($appDir, $badBinDir, $tmpDir, $scriptName); + $this->fail('Should have not been able to find BatchMake, but did, testCompileBatchMakeScript'); + } + catch(Zend_Exception $ze) + { + // correct behavior + } + + // this one should work + $this->kwBatchmakeComponent->compileBatchMakeScript($appDir, $binDir, $tmpDir, $scriptName); + + // now try a script that doesn't compile + $scriptName = 'CompileErrors.bms'; + $bmScripts = $this->kwBatchmakeComponent->findAndSymLinkDependentBatchmakeScripts($scriptDir,$tmpDir,$scriptName); + $bmms = $this->kwBatchmakeComponent->findAndSymLinkDependentBmms($appDir, $tmpDir, $bmScripts); + try + { + $this->kwBatchmakeComponent->compileBatchMakeScript($appDir, $binDir, $tmpDir, $scriptName); + $this->fail('Should have had a compile error but did not, testCompileBatchMakeScript'); + } + catch(Zend_Exception $ze) + { + // correct behavior + } + + } + + public function xtestGenerateCondorDag() + { + // create a task + $modelLoad = new MIDAS_ModelLoader(); + $batchmakeTaskModel = $modelLoad->loadModel('Task', 'batchmake'); + $usersFile = $this->loadData('User', 'default'); + $userDao = $this->User->load($usersFile[0]->getKey()); + $taskDao = $batchmakeTaskModel->createTask($userDao); + $userId = $taskDao->getUserId(); + $taskId = $taskDao->getKey(); + $subdirs = array(MIDAS_BATCHMAKE_SSP_DIR, $userId, $taskId); + // create a tmpDir based on the task and user + $applicationConfig = $this->kwBatchmakeComponent->getApplicationConfigProperties(); + $tmpDir = $this->kwBatchmakeComponent->createSubDirectories($applicationConfig[MIDAS_BATCHMAKE_TMP_DIR_PROPERTY] . "/", $subdirs); + + $scriptDir = $applicationConfig[MIDAS_BATCHMAKE_SCRIPT_DIR_PROPERTY]; + $binDir = $applicationConfig[MIDAS_BATCHMAKE_BIN_DIR_PROPERTY]; + $appDir = $applicationConfig[MIDAS_BATCHMAKE_APP_DIR_PROPERTY]; + + // a script that compiles + $scriptName = 'Compiles.bms'; + $bmScripts = $this->kwBatchmakeComponent->findAndSymLinkDependentBatchmakeScripts($scriptDir,$tmpDir,$scriptName); + $bmms = $this->kwBatchmakeComponent->findAndSymLinkDependentBmms($appDir, $tmpDir, $bmScripts); + + // try to generate the Condor script + $dagJobFile = $this->kwBatchmakeComponent->generateCondorDag($appDir, $tmpDir, $binDir, $scriptName); + $this->assertEquals($dagJobFile,'Compiles.bms.dagjob'); + // check that dag files and condor job files were created + $condorFiles = array($dagJobFile,'Compiles.1.bms.dagjob','Compiles.3.bms.dagjob','Compiles.5.bms.dagjob'); + foreach($condorFiles as $condorFile) + { + $this->assertFileExists($tmpDir.'/'.$condorFile); + } + // now look for some specific strings + $contents = file_get_contents($tmpDir.'/'. 'Compiles.bms.dagjob'); + $dagjobStrings = array('Job job3', 'Job job5','PARENT job1 CHILD job3', 'PARENT job3 CHILD job5'); + foreach($dagjobStrings as $string) + { + $this->assertTrue(preg_match("/".$string."/", $contents, $matches) === 1); + } + } + + + public function xtestCondorSubmitDag() + { + // create a task + $modelLoad = new MIDAS_ModelLoader(); + $batchmakeTaskModel = $modelLoad->loadModel('Task', 'batchmake'); + $usersFile = $this->loadData('User', 'default'); + $userDao = $this->User->load($usersFile[0]->getKey()); + $taskDao = $batchmakeTaskModel->createTask($userDao); + $userId = $taskDao->getUserId(); + $taskId = $taskDao->getKey(); + $subdirs = array(MIDAS_BATCHMAKE_SSP_DIR, $userId, $taskId); + // create a tmpDir based on the task and user + $applicationConfig = $this->kwBatchmakeComponent->getApplicationConfigProperties(); + $tmpDir = $this->kwBatchmakeComponent->createSubDirectories($applicationConfig[MIDAS_BATCHMAKE_TMP_DIR_PROPERTY] . "/", $subdirs); + + $scriptDir = $applicationConfig[MIDAS_BATCHMAKE_SCRIPT_DIR_PROPERTY]; + $binDir = $applicationConfig[MIDAS_BATCHMAKE_BIN_DIR_PROPERTY]; + $appDir = $applicationConfig[MIDAS_BATCHMAKE_APP_DIR_PROPERTY]; + $condorBinDir = $applicationConfig[MIDAS_BATCHMAKE_CONDOR_BIN_DIR_PROPERTY]; + + // a script that compiles + $scriptName = 'Compiles.bms'; + $bmScripts = $this->kwBatchmakeComponent->findAndSymLinkDependentBatchmakeScripts($scriptDir,$tmpDir,$scriptName); + $bmms = $this->kwBatchmakeComponent->findAndSymLinkDependentBmms($appDir, $tmpDir, $bmScripts); + + $dagScript = $this->kwBatchmakeComponent->generateCondorDag($appDir, $tmpDir, $binDir, $scriptName); + $this->kwBatchmakeComponent->condorSubmitDag($condorBinDir, $tmpDir, $dagScript); + // how to check this now? + } +/* + public function testSubmitCondorJob() + { + // create a task + $modelLoad = new MIDAS_ModelLoader(); + $batchmakeTaskModel = $modelLoad->loadModel('Task', 'batchmake'); + $usersFile = $this->loadData('User', 'default'); + $userDao = $this->User->load($usersFile[0]->getKey()); + $taskDao = $batchmakeTaskModel->createTask($userDao); + $userId = $taskDao->getUserId(); + $taskId = $taskDao->getKey(); + $subdirs = array(MIDAS_BATCHMAKE_SSP_DIR, $userId, $taskId); + // create a tmpDir based on the task and user + $applicationConfig = $this->kwBatchmakeComponent->getApplicationConfigProperties(); + $tmpDir = $this->kwBatchmakeComponent->createSubDirectories($applicationConfig[MIDAS_BATCHMAKE_TMP_DIR_PROPERTY] . "/", $subdirs); + + $scriptDir = $applicationConfig[MIDAS_BATCHMAKE_SCRIPT_DIR_PROPERTY]; + $binDir = $applicationConfig[MIDAS_BATCHMAKE_BIN_DIR_PROPERTY]; + $appDir = $applicationConfig[MIDAS_BATCHMAKE_APP_DIR_PROPERTY]; + $condorBinDir = $applicationConfig[MIDAS_BATCHMAKE_CONDOR_BIN_DIR_PROPERTY]; + + // a script that compiles + $scriptName = 'Compiles.bms'; + $bmScripts = $this->kwBatchmakeComponent->findAndSymLinkDependentBatchmakeScripts($scriptDir,$tmpDir,$scriptName); + $bmms = $this->kwBatchmakeComponent->findAndSymLinkDependentBmms($appDir, $tmpDir, $bmScripts); + + $submitFile = $this->kwBatchmakeComponent->generateCondorDagSubmit($condorBinDir, $appDir, $tmpDir, $binDir, $scriptName); + $this->kwBatchmakeComponent->submitCondorJob($condorBinDir, $tmpDir, $submitFile); + // now what to check for? + } + */ } diff --git a/modules/batchmake/tests/controllers/components/CMakeLists.txt b/modules/batchmake/tests/controllers/components/CMakeLists.txt new file mode 100644 index 000000000..62666b65e --- /dev/null +++ b/modules/batchmake/tests/controllers/components/CMakeLists.txt @@ -0,0 +1,6 @@ + + +add_midas_test( KWBatchmakeComponent KWBatchmakeComponentTest.php ) + +# Style +add_midas_style_test( StyleBatchmakeComponents ${CMAKE_SOURCE_DIR}/modules/batchmake/controllers/components ) \ No newline at end of file diff --git a/modules/batchmake/tests/controllers/components/KWBatchmakeComponentTest.php b/modules/batchmake/tests/controllers/components/KWBatchmakeComponentTest.php new file mode 100644 index 000000000..16a69d765 --- /dev/null +++ b/modules/batchmake/tests/controllers/components/KWBatchmakeComponentTest.php @@ -0,0 +1,407 @@ +kwBatchmakeComponent = new Batchmake_KWBatchmakeComponent(); + $this->kwBatchmakeComponent->setAlternateConfig(BASE_PATH.'/modules/batchmake/tests/configs/module.local.ini'); + $this->applicationConfig = $this->kwBatchmakeComponent->getApplicationConfigProperties(); + } + + /** set up tests*/ + public function setUp() + { + $this->setupDatabase(array('default')); + $this->_models = array('User'); + $this->enabledModules = array('batchmake'); + parent::setUp(); + } + + + // TEST KWBATCHMAKE COMPONENTS + // want a better way of doing this, but here is a start + + /** + * tests config setup, relies on an alternate testing config to be defined, + * these properties should all point to the batchmake module testfiles dirs. + */ + // should put the kwBatchmakeComponent as an instance variable? + // hopefully that would remove all of the redundant calls to setting it up + public function testIsConfigCorrect() + { + $this->assertTrue($this->kwBatchmakeComponent->isConfigCorrect()); + } + + /** + * tests that all the bmScripts that have been entered for testing are found + */ + public function testGetBatchmakeScripts() + { + $foundTestScripts = $this->kwBatchmakeComponent->getBatchmakeScripts(); + sort($foundTestScripts); + $expectedTestScripts = array("Compiles.bms", "Myscript2.bms", "noscripts.bms", + "anotherscript.bms", "anotherscriptwitherrors.bms", "bmmswitherrors.bms", + "cycle1.bms", "cycle31.bms", "cycle32.bms", "cycle33.bms", "nocycle1.bms", + "nocycle2.bms", "nocycle3.bms", "myscript.bms", "PixelCounter.bms", + "CompileErrors.bms"); + sort($expectedTestScripts); + $this->assertEquals($foundTestScripts, $expectedTestScripts); + } + + /** + * @TODO some model testing, and better ways of loading the models + * test creating a model and creating the right subdirs + */ + + + + /** + * helper function to clear out any files in a directory + */ + protected function clearDirFiles($dirToClear) + { + foreach(scandir($dirToClear) as $filename) + { + if($filename && $filename != '.' && $filename != '..') + { + unlink($dirToClear.'/'.$filename); + } + } + } + + + /** + * helper function to run a test case + */ + protected function fASLDBSWCDtestcase($scriptDir, $tmpDir, $scriptName, $expectedSet) + { + // clear the directory of any existing files + $this->clearDirFiles($tmpDir); + + // try symlinking all the batchmake files starting with $scriptName + $bmScriptsProcessed = $this->kwBatchmakeComponent->findAndSymLinkDependentBatchmakeScriptsWithCycleDetection($scriptDir, $tmpDir, $scriptName); + + // check that the correct batchmake scripts are there, and only those + // easiest just to add '.' and '..' to expected list + $expectedSet[] = '..'; + $expectedSet[] = '.'; + sort($expectedSet); + + $foundScripts = scandir($tmpDir); + sort($foundScripts); + + $this->assertEquals($expectedSet, $foundScripts, + "Expected batchmake scripts not found rooted from ".$scriptName); + + // add in '.' and '..' + $bmScriptsProcessed[] = '.'; + $bmScriptsProcessed[] = '..'; + sort($bmScriptsProcessed); + + // also check that the set of scripts returned from the method is this same set + $this->assertEquals($expectedSet, $bmScriptsProcessed, + "Expected batchmake scripts not equal to those returned from processing ".$scriptName); + } + + /** + * helper function to run a test case that is expected to throw an exception + */ + protected function fASLDBSWCDtestcaseException($scriptDir, $tmpDir, $scriptName) + { + try + { + // need to suppress error output to keep test from failing, despite exception being caught + $bmScripts = @$this->kwBatchmakeComponent->findAndSymLinkDependentBatchmakeScriptsWithCycleDetection($scriptDir, $tmpDir, $scriptName); + $this->fail('Expected an exception for $scriptName, but did not get one.'); + } + catch(Zend_Exception $ze) + { + // if we got here, this is the correct behavior + $this->assertTrue(true); + } + } + + + /** tests findAndSymLinkDependentBatchmakeScriptsWithCycleDetection. */ + public function testFindAndSymLinkDependentBatchmakeScriptsWithCycleDetection() + { + $applicationConfig = $this->kwBatchmakeComponent->getApplicationConfigProperties(); + // create a task + $modelLoad = new MIDAS_ModelLoader(); + $batchmakeTaskModel = $modelLoad->loadModel('Task', 'batchmake'); + $usersFile = $this->loadData('User', 'default'); + $userDao = $this->User->load($usersFile[0]->getKey()); + $taskDao = $batchmakeTaskModel->createTask($userDao); + $userId = $taskDao->getUserId(); + $taskId = $taskDao->getKey(); + $subdirs = array(MIDAS_BATCHMAKE_SSP_DIR, $userId, $taskId); + // create a tmpDir based on the task and user + $tmpDir = KWUtils::createSubDirectories($applicationConfig[MIDAS_BATCHMAKE_TMP_DIR_PROPERTY] . "/", $subdirs); + + $scriptDir = $applicationConfig[MIDAS_BATCHMAKE_SCRIPT_DIR_PROPERTY]; + + + $scriptName = 'anotherscript.bms'; + $expectedSet = array("myscript.bms", "Myscript2.bms", + "anotherscript.bms", "noscripts.bms", "PixelCounter.bms"); + $this->fASLDBSWCDtestcase($scriptDir, $tmpDir, $scriptName, $expectedSet); + + $scriptName = "noscripts.bms"; + $expectedSet = array("noscripts.bms"); + $this->fASLDBSWCDtestcase($scriptDir, $tmpDir, $scriptName, $expectedSet); + + // try symlinking all the batchmake files starting with anotherscriptwitherrors.bms + // expect an exception, as this script includes a non-existent script + $scriptName = 'anotherscriptwitherrors.bms'; + $this->fASLDBSWCDtestcaseException($scriptDir, $tmpDir, $scriptName); + + // cycle detection tests + + // check a script with no cycle,1->2, 1->3, 3->2 + // clear the directory of the symlinked files + $scriptName = "nocycle1.bms"; + $expectedSet = array("nocycle1.bms", "nocycle2.bms", "nocycle3.bms"); + $this->fASLDBSWCDtestcase($scriptDir, $tmpDir, $scriptName, $expectedSet); + + // expect an exception, as this script has a simple cycle + // 1->1 + $scriptName = 'cycle1.bms'; + $this->fASLDBSWCDtestcaseException($scriptDir, $tmpDir, $scriptName); + + // check a script with a more complex cycle, 1->2, 1->3, 2->3, 3->2 + $scriptName = 'cycle31.bms'; + $this->fASLDBSWCDtestcaseException($scriptDir, $tmpDir, $scriptName); + } + + + /** tests findAndSymLinkDependentBmms */ + public function testFindAndSymLinkDependentBmms() + { + $applicationConfig = $this->kwBatchmakeComponent->getApplicationConfigProperties(); + // create a task + $modelLoad = new MIDAS_ModelLoader(); + $batchmakeTaskModel = $modelLoad->loadModel('Task', 'batchmake'); + $usersFile = $this->loadData('User', 'default'); + $userDao = $this->User->load($usersFile[0]->getKey()); + $taskDao = $batchmakeTaskModel->createTask($userDao); + $userId = $taskDao->getUserId(); + $taskId = $taskDao->getKey(); + $subdirs = array(MIDAS_BATCHMAKE_SSP_DIR, $userId, $taskId); + // create a tmpDir based on the task and user + $tmpDir = KWUtils::createSubDirectories($applicationConfig[MIDAS_BATCHMAKE_TMP_DIR_PROPERTY] . "/", $subdirs); + + $scriptDir = $applicationConfig[MIDAS_BATCHMAKE_SCRIPT_DIR_PROPERTY]; + $appDir = $applicationConfig[MIDAS_BATCHMAKE_APP_DIR_PROPERTY]; + + // now try symlinking all the batchmake files starting with anotherscript.bms + $scriptName = 'anotherscript.bms'; + $bmScripts_anotherscript = $this->kwBatchmakeComponent->findAndSymLinkDependentBatchmakeScriptsWithCycleDetection($scriptDir, $tmpDir, $scriptName); + + + $bmms = $this->kwBatchmakeComponent->findAndSymLinkDependentBmms($appDir, $tmpDir, $bmScripts_anotherscript); + // these come as [ name of app => script where found ] + // convert them to a form useful for comparison + $processedBmms_anotherscript = array(); + foreach($bmms as $bmm => $script) + { + $processedBmms_anotherscript[] = $bmm.'.bmm'; + } + sort($processedBmms_anotherscript); + + $globOutput = glob($tmpDir.'/*.bmm'); + // strip off the path + $foundBmms_anotherscript = array(); + foreach($globOutput as $bmm) + { + $foundBmms_anotherscript[] = basename($bmm); + } + sort($foundBmms_anotherscript); + + $expectedBmms_anotherscript = array("AnotherApp.bmm", "MyApp2.bmm", + "PixelCounter.bmm", "TestApp1.bmm", "TestApp2.bmm", "myapp.bmm"); + sort($expectedBmms_anotherscript); + + // compare the three arrays + $this->assertEquals($processedBmms_anotherscript, $expectedBmms_anotherscript, "BMMs: processed != expected, for anotherscript.bms"); + $this->assertEquals($processedBmms_anotherscript, $foundBmms_anotherscript, "BMMs: processed != found, for anotherscript.bms"); + + } + + + + + + + + /** tests testCompileBatchMakeScript */ + public function testCompileBatchMakeScript() + { + // create a task + $modelLoad = new MIDAS_ModelLoader(); + $batchmakeTaskModel = $modelLoad->loadModel('Task', 'batchmake'); + $usersFile = $this->loadData('User', 'default'); + $userDao = $this->User->load($usersFile[0]->getKey()); + $taskDao = $batchmakeTaskModel->createTask($userDao); + $userId = $taskDao->getUserId(); + $taskId = $taskDao->getKey(); + $subdirs = array(MIDAS_BATCHMAKE_SSP_DIR, $userId, $taskId); + // create a tmpDir based on the task and user + $applicationConfig = $this->kwBatchmakeComponent->getApplicationConfigProperties(); + $tmpDir = KWUtils::createSubDirectories($applicationConfig[MIDAS_BATCHMAKE_TMP_DIR_PROPERTY] . "/", $subdirs); + + $scriptDir = $applicationConfig[MIDAS_BATCHMAKE_SCRIPT_DIR_PROPERTY]; + $binDir = $applicationConfig[MIDAS_BATCHMAKE_BIN_DIR_PROPERTY]; + $appDir = $applicationConfig[MIDAS_BATCHMAKE_APP_DIR_PROPERTY]; + + // a script that compiles + $scriptName = 'Compiles.bms'; + $bmScripts = $this->kwBatchmakeComponent->findAndSymLinkDependentBatchmakeScriptsWithCycleDetection($scriptDir, $tmpDir, $scriptName); + $bmms = $this->kwBatchmakeComponent->findAndSymLinkDependentBmms($appDir, $tmpDir, $bmScripts); + + // first try with a bad path to BatchMake + $badBinDir = '/a/dir/not/likely/to/exist'; + try + { + $this->kwBatchmakeComponent->compileBatchMakeScript($appDir, $badBinDir, $tmpDir, $scriptName); + $this->fail('Should have not been able to find BatchMake, but did, testCompileBatchMakeScript'); + } + catch(Zend_Exception $ze) + { + // if we got here, this is the correct behavior + $this->assertTrue(true); + } + + // this one should work + $this->kwBatchmakeComponent->compileBatchMakeScript($appDir, $binDir, $tmpDir, $scriptName); + + // now try a script that doesn't compile + $scriptName = 'CompileErrors.bms'; + $bmScripts = $this->kwBatchmakeComponent->findAndSymLinkDependentBatchmakeScriptsWithCycleDetection($scriptDir, $tmpDir, $scriptName); + $bmms = $this->kwBatchmakeComponent->findAndSymLinkDependentBmms($appDir, $tmpDir, $bmScripts); + try + { + $this->kwBatchmakeComponent->compileBatchMakeScript($appDir, $binDir, $tmpDir, $scriptName); + $this->fail('Should have had a compile error but did not, testCompileBatchMakeScript'); + } + catch(Zend_Exception $ze) + { + // if we got here, this is the correct behavior + $this->assertTrue(true); + } + + } + + /** tests generateCondorDag */ + public function testGenerateCondorDag() + { + // create a task + $modelLoad = new MIDAS_ModelLoader(); + $batchmakeTaskModel = $modelLoad->loadModel('Task', 'batchmake'); + $usersFile = $this->loadData('User', 'default'); + $userDao = $this->User->load($usersFile[0]->getKey()); + $taskDao = $batchmakeTaskModel->createTask($userDao); + $userId = $taskDao->getUserId(); + $taskId = $taskDao->getKey(); + $subdirs = array(MIDAS_BATCHMAKE_SSP_DIR, $userId, $taskId); + // create a tmpDir based on the task and user + $applicationConfig = $this->kwBatchmakeComponent->getApplicationConfigProperties(); + $tmpDir = KWUtils::createSubDirectories($applicationConfig[MIDAS_BATCHMAKE_TMP_DIR_PROPERTY] . "/", $subdirs); + + $scriptDir = $applicationConfig[MIDAS_BATCHMAKE_SCRIPT_DIR_PROPERTY]; + $binDir = $applicationConfig[MIDAS_BATCHMAKE_BIN_DIR_PROPERTY]; + $appDir = $applicationConfig[MIDAS_BATCHMAKE_APP_DIR_PROPERTY]; + + // a script that compiles + $scriptName = 'Compiles.bms'; + $bmScripts = $this->kwBatchmakeComponent->findAndSymLinkDependentBatchmakeScriptsWithCycleDetection($scriptDir, $tmpDir, $scriptName); + $bmms = $this->kwBatchmakeComponent->findAndSymLinkDependentBmms($appDir, $tmpDir, $bmScripts); + + // try to generate the Condor script + $dagJobFile = $this->kwBatchmakeComponent->generateCondorDag($appDir, $tmpDir, $binDir, $scriptName); + $this->assertEquals($dagJobFile, 'Compiles.bms.dagjob'); + // check that dag files and condor job files were created + $condorFiles = array($dagJobFile, 'Compiles.1.bms.dagjob', 'Compiles.3.bms.dagjob', 'Compiles.5.bms.dagjob'); + foreach($condorFiles as $condorFile) + { + $this->assertFileExists($tmpDir.'/'.$condorFile); + } + // now look for some specific strings + $contents = file_get_contents($tmpDir.'/'. 'Compiles.bms.dagjob'); + $dagjobStrings = array('Job job3', 'Job job5', 'PARENT job1 CHILD job3', 'PARENT job3 CHILD job5'); + foreach($dagjobStrings as $string) + { + $this->assertTrue(preg_match("/".$string."/", $contents, $matches) === 1); + } + } + + + /** tests function testCondorSubmitDag */ + public function testCondorSubmitDag() + { + // create a task + $modelLoad = new MIDAS_ModelLoader(); + $batchmakeTaskModel = $modelLoad->loadModel('Task', 'batchmake'); + $usersFile = $this->loadData('User', 'default'); + $userDao = $this->User->load($usersFile[0]->getKey()); + $taskDao = $batchmakeTaskModel->createTask($userDao); + $userId = $taskDao->getUserId(); + $taskId = $taskDao->getKey(); + $subdirs = array(MIDAS_BATCHMAKE_SSP_DIR, $userId, $taskId); + // create a tmpDir based on the task and user + $applicationConfig = $this->kwBatchmakeComponent->getApplicationConfigProperties(); + $tmpDir = KWUtils::createSubDirectories($applicationConfig[MIDAS_BATCHMAKE_TMP_DIR_PROPERTY] . "/", $subdirs); + + $scriptDir = $applicationConfig[MIDAS_BATCHMAKE_SCRIPT_DIR_PROPERTY]; + $binDir = $applicationConfig[MIDAS_BATCHMAKE_BIN_DIR_PROPERTY]; + $appDir = $applicationConfig[MIDAS_BATCHMAKE_APP_DIR_PROPERTY]; + $condorBinDir = $applicationConfig[MIDAS_BATCHMAKE_CONDOR_BIN_DIR_PROPERTY]; + + // a script that compiles + $scriptName = 'Compiles.bms'; + $bmScripts = $this->kwBatchmakeComponent->findAndSymLinkDependentBatchmakeScriptsWithCycleDetection($scriptDir, $tmpDir, $scriptName); + $bmms = $this->kwBatchmakeComponent->findAndSymLinkDependentBmms($appDir, $tmpDir, $bmScripts); + + $dagScript = $this->kwBatchmakeComponent->generateCondorDag($appDir, $tmpDir, $binDir, $scriptName); + $this->kwBatchmakeComponent->condorSubmitDag($condorBinDir, $tmpDir, $dagScript); + // how to check this now? + } + + + + + + + + + + + + + + + + } // end class diff --git a/modules/batchmake/tests/testfiles/midas3batchmake/bin/AnotherApp.bmm b/modules/batchmake/tests/testfiles/midas3batchmake/bin/AnotherApp.bmm new file mode 100644 index 000000000..e69de29bb diff --git a/modules/batchmake/tests/testfiles/midas3batchmake/bin/MyApp2.bmm b/modules/batchmake/tests/testfiles/midas3batchmake/bin/MyApp2.bmm new file mode 100644 index 000000000..e69de29bb diff --git a/modules/batchmake/tests/testfiles/midas3batchmake/bin/PixelCounter.bmm b/modules/batchmake/tests/testfiles/midas3batchmake/bin/PixelCounter.bmm new file mode 100644 index 000000000..65dfbc38c --- /dev/null +++ b/modules/batchmake/tests/testfiles/midas3batchmake/bin/PixelCounter.bmm @@ -0,0 +1,58 @@ + + 1.0 + + PixelCounter + Not defined + SET_PATH_TO/PixelCounter + + + 4 + filename + + 0 + 1 + 0 + + + 3 + threshold + + 0 + 0 + 0 + + + 1 + GenerateMetaOutput + -generateMetaOutput + 0 + 0 + 1 + + + 1 + GenerateXMLMetaOutput + -oxml + 0 + 0 + 1 + + + 1 + GenerateXMLFile + -ofxml + 0 + 0 + 1 + + + 4 + GenerateXMLFile.GenerateXMLFile + + 3 + 2 + 0 + + + + diff --git a/modules/batchmake/tests/testfiles/midas3batchmake/bin/TestApp1.bmm b/modules/batchmake/tests/testfiles/midas3batchmake/bin/TestApp1.bmm new file mode 100644 index 000000000..e69de29bb diff --git a/modules/batchmake/tests/testfiles/midas3batchmake/bin/TestApp2.bmm b/modules/batchmake/tests/testfiles/midas3batchmake/bin/TestApp2.bmm new file mode 100644 index 000000000..e69de29bb diff --git a/modules/batchmake/tests/testfiles/midas3batchmake/bin/myapp.bmm b/modules/batchmake/tests/testfiles/midas3batchmake/bin/myapp.bmm new file mode 100644 index 000000000..e69de29bb diff --git a/modules/batchmake/tests/testfiles/midas3batchmake/data/testing/sphere_10.mha b/modules/batchmake/tests/testfiles/midas3batchmake/data/testing/sphere_10.mha new file mode 100644 index 0000000000000000000000000000000000000000..32ce4e1119676bc0fd7d97be976f3ac781a9fe67 GIT binary patch literal 816 zcmeZI%1TWx38^edRj^g?%uP&B)x!{7gx&saRx(#x5%$k_GWH5bL53~hmZ%nZbV z6TOC)J}aL-(rI|f!+p`?ll_0bGcde+|7)v>o7BV=d+zx&Z=K7*pu%Ntv~~5;*I~yb zGFAL~Cc0S($@<^hB7B#R$zjQjE#I!1bw-vi+HJ=W7L;=^=&Il5^~d6c{?C|EsvcF# z;G_Fy_Kqv(M4r1!uS=eOq;vX#+aB&o=gLh#wskX~4=9ph=u^$RDrI)qJE5y~t5?$J ztIwbDHB3snKgH{8+vFcQpY#$AGj7lt#^$61)o zIdWadOork1YVkf^^^aRyc-a{;Ymdl^UVm`2=$q}GyF&ZgPoHBx;Jh*4YiZhH&l<5x z@eL0d9&C3!^f>bAoRtTEuiF<|_Gr%gtVgb5?DG~rv0#|8M6`67h5oIA?lt>Tg>3Jy zI?A{`i{Z_>8%;}Bt^KpwRc1#<>YLdwez58qF`QVc8OWXm#^2?AU*(*Jp?1y7+0joqg)|cPbc(iK3Xt?57o_j{yL; C_%-qX literal 0 HcmV?d00001 diff --git a/modules/batchmake/tests/testfiles/midas3batchmake/data/testing/sphere_15.mha b/modules/batchmake/tests/testfiles/midas3batchmake/data/testing/sphere_15.mha new file mode 100644 index 0000000000000000000000000000000000000000..90100d3d1da7aa7d14b4afa3185bdaae054ecba3 GIT binary patch literal 1250 zcmeZI%1TWx38^edRj^g?%uP&B)`&nE0h=U;Xbj*;1XtuV=ZXd2XrL z)w?a@-%ZwrxT*1Tw!d9nk-BETC(pEu%{tLshBb4 zE9nZ=+O1w$mUQzKcR%xmn7B8o+ubjUmY!BR{r$GejSH z=gGZpUzzTvtmanFQ}!wxNH+LKd0bG3{(V5ODi> zv9MW;$cw7OOP@2EEqN*|9#h-Nx8q?Mv)D|R(=O{*=dS{rU(UEKq{Kh);NIRNzI!Ei zd=tN6%_!}+=<^X5X+8fbYwV=?Bs^Wei-^wQEf0L8U~uKsgdN6-hg}=jo|*l5{lxE$ zN86a!1zxmuWlcXEJH>*7XRDFQJD(c`feG55pZ&>xQk$^xVM8|9hM%@~TtP0=c7AjY zX!^PL4;kW?gig8^3U=2TyH07Ly?UE$46n9zTYq$a_N21GH*spn*XY2`TLu2Ad`!-{ z^X-Id_Z3U)@%%YA$KtcJJDR;M%({FEAjr#MH^?mZkUL{=l%V3_piC||Q08LpLU;qFB literal 0 HcmV?d00001 diff --git a/modules/batchmake/tests/testfiles/midas3batchmake/data/testing/sphere_5.mha b/modules/batchmake/tests/testfiles/midas3batchmake/data/testing/sphere_5.mha new file mode 100644 index 0000000000000000000000000000000000000000..5a5fd9c4ba032f072e3d3b5c0f7d2188296d5336 GIT binary patch literal 527 zcmeZI%1TWx38^edRj^g?%uP&BXR7z4xh zjQKvH<}-tK*fA)m)Te~rP4Rm8i=jb;f4a7f48w#+4~tscnHhp?V%B{qU~uSaKP@W9 g$FRfw->+jLeLM_5HvT*5qe-&m6*t&E$UN2t0E^a=Qvd(} literal 0 HcmV?d00001 diff --git a/modules/batchmake/tests/testfiles/midas3batchmake/script/CompileErrors.bms b/modules/batchmake/tests/testfiles/midas3batchmake/script/CompileErrors.bms new file mode 100644 index 000000000..f5034a295 --- /dev/null +++ b/modules/batchmake/tests/testfiles/midas3batchmake/script/CompileErrors.bms @@ -0,0 +1,7 @@ +# Should the script end in case of errors +ExitOnError(1) + +Echo('This script should not compile') + +Foreach(var ${varlist}) +Endforeach() diff --git a/modules/batchmake/tests/testfiles/midas3batchmake/script/Compiles.bms b/modules/batchmake/tests/testfiles/midas3batchmake/script/Compiles.bms new file mode 100644 index 000000000..993eceee3 --- /dev/null +++ b/modules/batchmake/tests/testfiles/midas3batchmake/script/Compiles.bms @@ -0,0 +1,22 @@ +# Should the script end in case of errors +ExitOnError(1) + +# all params are hardcoded, just used to check for creating a dagjob and condor scripts + +SetApp(pixelCounter @PixelCounter) +SetAppOption(pixelCounter.threshold 10) +SetAppOption(pixelCounter.GenerateXMLFile 1) + +Set(hardcodedlist 'sphere_5.mha' 'sphere_10.mha' 'sphere_15.mha') + +# Generic script options +Foreach(file ${hardcodedlist}) + GetFilename(stem ${file} NAME_WITHOUT_EXTENSION) + + SetAppOption(pixelCounter.filename ${file}) + SetAppOption(pixelCounter.GenerateXMLFile.GenerateXMLFile ${stem}.xml) + + # Run the application + Run(prog_output ${pixelCounter}) + +Endforeach(file) diff --git a/modules/batchmake/tests/testfiles/midas3batchmake/script/Myscript2.bms b/modules/batchmake/tests/testfiles/midas3batchmake/script/Myscript2.bms new file mode 100644 index 000000000..7060b5967 --- /dev/null +++ b/modules/batchmake/tests/testfiles/midas3batchmake/script/Myscript2.bms @@ -0,0 +1,25 @@ +# Should the script end in case of errors +ExitOnError(1) + +# Imported parameters: cfg_input_directory, cfg_output_directory, cfg_application_directory, cfg_threshold +Include(PixelCounter.config.bms) + +# Application setup +SetApp(pixelCounter @PixelCounter) +SetApp(myapp2 @MyApp2) +SetAppOption(pixelCounter.threshold ${cfg_threshold}) +SetAppOption(pixelCounter.GenerateXMLFile 1) + + +# Generic script options +Foreach(file ${cfg_input_cart}) + GetFilename(stem ${file} NAME_WITHOUT_EXTENSION) + + SetAppOption(pixelCounter.filename ${file}) + SetAppOption(pixelCounter.GenerateXMLFile.GenerateXMLFile ${stem}.${cfg_output1}.xml) + + # Run the application + CondorPostScript(pixelCounter ${cfg_condorpostscript} ${cfg_midas_baseURL} ${cfg_itemid} ${cfg_email} ${cfg_appname} ${cfg_apikey} ${cfg_output_directory} ${stem}.${cfg_output1}.xml) + Run(prog_output ${pixelCounter}) + +Endforeach(file) diff --git a/modules/batchmake/tests/testfiles/midas3batchmake/script/PixelCounter.bms b/modules/batchmake/tests/testfiles/midas3batchmake/script/PixelCounter.bms new file mode 100644 index 000000000..8c4b44a89 --- /dev/null +++ b/modules/batchmake/tests/testfiles/midas3batchmake/script/PixelCounter.bms @@ -0,0 +1,24 @@ +# Should the script end in case of errors +ExitOnError(1) + +# Imported parameters: cfg_input_directory, cfg_output_directory, cfg_application_directory, cfg_threshold +Include(PixelCounter.config.bms) + +# Application setup +SetApp(pixelCounter @PixelCounter) +SetAppOption(pixelCounter.threshold ${cfg_threshold}) +SetAppOption(pixelCounter.GenerateXMLFile 1) + + +# Generic script options +Foreach(file ${cfg_input_cart}) + GetFilename(stem ${file} NAME_WITHOUT_EXTENSION) + + SetAppOption(pixelCounter.filename ${file}) + SetAppOption(pixelCounter.GenerateXMLFile.GenerateXMLFile ${stem}.${cfg_output1}.xml) + + # Run the application + CondorPostScript(pixelCounter ${cfg_condorpostscript} ${cfg_midas_baseURL} ${cfg_itemid} ${cfg_email} ${cfg_appname} ${cfg_apikey} ${cfg_output_directory} ${stem}.${cfg_output1}.xml) + Run(prog_output ${pixelCounter}) + +Endforeach(file) diff --git a/modules/batchmake/tests/testfiles/midas3batchmake/script/anotherscript.bms b/modules/batchmake/tests/testfiles/midas3batchmake/script/anotherscript.bms new file mode 100644 index 000000000..52f80ca78 --- /dev/null +++ b/modules/batchmake/tests/testfiles/midas3batchmake/script/anotherscript.bms @@ -0,0 +1,28 @@ +# Should the script end in case of errors +ExitOnError(1) + +# Imported parameters: cfg_input_directory, cfg_output_directory, cfg_application_directory, cfg_threshold +Include(PixelCounter.config.bms) +Include(PixelCounter.bms) +Include(myscript.bms) +Include(Myscript2.bms) + +# Application setup +SetApp(pixelCounter @PixelCounter) +SetApp(anotherapp @AnotherApp) +SetAppOption(pixelCounter.threshold ${cfg_threshold}) +SetAppOption(pixelCounter.GenerateXMLFile 1) + + +# Generic script options +Foreach(file ${cfg_input_cart}) + GetFilename(stem ${file} NAME_WITHOUT_EXTENSION) + + SetAppOption(pixelCounter.filename ${file}) + SetAppOption(pixelCounter.GenerateXMLFile.GenerateXMLFile ${stem}.${cfg_output1}.xml) + + # Run the application + CondorPostScript(pixelCounter ${cfg_condorpostscript} ${cfg_midas_baseURL} ${cfg_itemid} ${cfg_email} ${cfg_appname} ${cfg_apikey} ${cfg_output_directory} ${stem}.${cfg_output1}.xml) + Run(prog_output ${pixelCounter}) + +Endforeach(file) diff --git a/modules/batchmake/tests/testfiles/midas3batchmake/script/anotherscriptwitherrors.bms b/modules/batchmake/tests/testfiles/midas3batchmake/script/anotherscriptwitherrors.bms new file mode 100644 index 000000000..3bac9507d --- /dev/null +++ b/modules/batchmake/tests/testfiles/midas3batchmake/script/anotherscriptwitherrors.bms @@ -0,0 +1,25 @@ +# Should the script end in case of errors +ExitOnError(1) + +# Imported parameters: cfg_input_directory, cfg_output_directory, cfg_application_directory, cfg_threshold +Include(nonexistentscript.bms) + +# Application setup +SetApp(pixelCounter @PixelCounter) +SetApp(anotherapp @AnotherApp) +SetAppOption(pixelCounter.threshold ${cfg_threshold}) +SetAppOption(pixelCounter.GenerateXMLFile 1) + + +# Generic script options +Foreach(file ${cfg_input_cart}) + GetFilename(stem ${file} NAME_WITHOUT_EXTENSION) + + SetAppOption(pixelCounter.filename ${file}) + SetAppOption(pixelCounter.GenerateXMLFile.GenerateXMLFile ${stem}.${cfg_output1}.xml) + + # Run the application + CondorPostScript(pixelCounter ${cfg_condorpostscript} ${cfg_midas_baseURL} ${cfg_itemid} ${cfg_email} ${cfg_appname} ${cfg_apikey} ${cfg_output_directory} ${stem}.${cfg_output1}.xml) + Run(prog_output ${pixelCounter}) + +Endforeach(file) diff --git a/modules/batchmake/tests/testfiles/midas3batchmake/script/bmmswitherrors.bms b/modules/batchmake/tests/testfiles/midas3batchmake/script/bmmswitherrors.bms new file mode 100644 index 000000000..095f1eb08 --- /dev/null +++ b/modules/batchmake/tests/testfiles/midas3batchmake/script/bmmswitherrors.bms @@ -0,0 +1,23 @@ +# Should the script end in case of errors +ExitOnError(1) + + +# Application setup +SetApp(testapp1 @NoApp) +SetApp(testapp2 @TestApp2) +SetAppOption(pixelCounter.threshold ${cfg_threshold}) +SetAppOption(pixelCounter.GenerateXMLFile 1) + + +# Generic script options +Foreach(file ${cfg_input_cart}) + GetFilename(stem ${file} NAME_WITHOUT_EXTENSION) + + SetAppOption(pixelCounter.filename ${file}) + SetAppOption(pixelCounter.GenerateXMLFile.GenerateXMLFile ${stem}.${cfg_output1}.xml) + + # Run the application + CondorPostScript(pixelCounter ${cfg_condorpostscript} ${cfg_midas_baseURL} ${cfg_itemid} ${cfg_email} ${cfg_appname} ${cfg_apikey} ${cfg_output_directory} ${stem}.${cfg_output1}.xml) + Run(prog_output ${pixelCounter}) + +Endforeach(file) diff --git a/modules/batchmake/tests/testfiles/midas3batchmake/script/cycle1.bms b/modules/batchmake/tests/testfiles/midas3batchmake/script/cycle1.bms new file mode 100644 index 000000000..11a3048ea --- /dev/null +++ b/modules/batchmake/tests/testfiles/midas3batchmake/script/cycle1.bms @@ -0,0 +1 @@ +Include(cycle1.bms) diff --git a/modules/batchmake/tests/testfiles/midas3batchmake/script/cycle31.bms b/modules/batchmake/tests/testfiles/midas3batchmake/script/cycle31.bms new file mode 100644 index 000000000..ca1f270a3 --- /dev/null +++ b/modules/batchmake/tests/testfiles/midas3batchmake/script/cycle31.bms @@ -0,0 +1,2 @@ +Include(cycle32.bms) +Include(cycle33.bms) diff --git a/modules/batchmake/tests/testfiles/midas3batchmake/script/cycle32.bms b/modules/batchmake/tests/testfiles/midas3batchmake/script/cycle32.bms new file mode 100644 index 000000000..0f5ca2cd0 --- /dev/null +++ b/modules/batchmake/tests/testfiles/midas3batchmake/script/cycle32.bms @@ -0,0 +1 @@ +Include(cycle33.bms) diff --git a/modules/batchmake/tests/testfiles/midas3batchmake/script/cycle33.bms b/modules/batchmake/tests/testfiles/midas3batchmake/script/cycle33.bms new file mode 100644 index 000000000..235b67832 --- /dev/null +++ b/modules/batchmake/tests/testfiles/midas3batchmake/script/cycle33.bms @@ -0,0 +1 @@ +Include(cycle32.bms) diff --git a/modules/batchmake/tests/testfiles/midas3batchmake/script/myscript.bms b/modules/batchmake/tests/testfiles/midas3batchmake/script/myscript.bms new file mode 100644 index 000000000..a3474c8f4 --- /dev/null +++ b/modules/batchmake/tests/testfiles/midas3batchmake/script/myscript.bms @@ -0,0 +1,26 @@ +# Should the script end in case of errors +ExitOnError(1) + +# Imported parameters: cfg_input_directory, cfg_output_directory, cfg_application_directory, cfg_threshold +Include(PixelCounter.config.bms) +Include(noscripts.bms) + +# Application setup +SetApp(pixelCounter @PixelCounter) +SetApp(myapp @myapp) +SetAppOption(pixelCounter.threshold ${cfg_threshold}) +SetAppOption(pixelCounter.GenerateXMLFile 1) + + +# Generic script options +Foreach(file ${cfg_input_cart}) + GetFilename(stem ${file} NAME_WITHOUT_EXTENSION) + + SetAppOption(pixelCounter.filename ${file}) + SetAppOption(pixelCounter.GenerateXMLFile.GenerateXMLFile ${stem}.${cfg_output1}.xml) + + # Run the application + CondorPostScript(pixelCounter ${cfg_condorpostscript} ${cfg_midas_baseURL} ${cfg_itemid} ${cfg_email} ${cfg_appname} ${cfg_apikey} ${cfg_output_directory} ${stem}.${cfg_output1}.xml) + Run(prog_output ${pixelCounter}) + +Endforeach(file) diff --git a/modules/batchmake/tests/testfiles/midas3batchmake/script/nocycle1.bms b/modules/batchmake/tests/testfiles/midas3batchmake/script/nocycle1.bms new file mode 100644 index 000000000..239a97411 --- /dev/null +++ b/modules/batchmake/tests/testfiles/midas3batchmake/script/nocycle1.bms @@ -0,0 +1,2 @@ +Include(nocycle2.bms) +Include(nocycle3.bms) diff --git a/modules/batchmake/tests/testfiles/midas3batchmake/script/nocycle2.bms b/modules/batchmake/tests/testfiles/midas3batchmake/script/nocycle2.bms new file mode 100644 index 000000000..a763c6d87 --- /dev/null +++ b/modules/batchmake/tests/testfiles/midas3batchmake/script/nocycle2.bms @@ -0,0 +1 @@ +Echo('nothing to say') diff --git a/modules/batchmake/tests/testfiles/midas3batchmake/script/nocycle3.bms b/modules/batchmake/tests/testfiles/midas3batchmake/script/nocycle3.bms new file mode 100644 index 000000000..a78bec42d --- /dev/null +++ b/modules/batchmake/tests/testfiles/midas3batchmake/script/nocycle3.bms @@ -0,0 +1 @@ +Include(nocycle2.bms) diff --git a/modules/batchmake/tests/testfiles/midas3batchmake/script/noscripts.bms b/modules/batchmake/tests/testfiles/midas3batchmake/script/noscripts.bms new file mode 100644 index 000000000..bdba1abd2 --- /dev/null +++ b/modules/batchmake/tests/testfiles/midas3batchmake/script/noscripts.bms @@ -0,0 +1,23 @@ +# Should the script end in case of errors +ExitOnError(1) + + +# Application setup +SetApp(testapp1 @TestApp1) +SetApp(testapp2 @TestApp2) +SetAppOption(pixelCounter.threshold ${cfg_threshold}) +SetAppOption(pixelCounter.GenerateXMLFile 1) + + +# Generic script options +Foreach(file ${cfg_input_cart}) + GetFilename(stem ${file} NAME_WITHOUT_EXTENSION) + + SetAppOption(pixelCounter.filename ${file}) + SetAppOption(pixelCounter.GenerateXMLFile.GenerateXMLFile ${stem}.${cfg_output1}.xml) + + # Run the application + CondorPostScript(pixelCounter ${cfg_condorpostscript} ${cfg_midas_baseURL} ${cfg_itemid} ${cfg_email} ${cfg_appname} ${cfg_apikey} ${cfg_output_directory} ${stem}.${cfg_output1}.xml) + Run(prog_output ${pixelCounter}) + +Endforeach(file) diff --git a/modules/batchmake/views/config/index.phtml b/modules/batchmake/views/config/index.phtml index 6f1e2fc5b..ccd9c96e7 100644 --- a/modules/batchmake/views/config/index.phtml +++ b/modules/batchmake/views/config/index.phtml @@ -17,7 +17,6 @@ $this->headScript()->appendFile($this->moduleWebroot . '/public/js/config/config
-
'; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 17557ff7c..fb5438b03 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -58,3 +58,4 @@ endfunction(add_midas_style_test) add_midas_style_test( StyleTestsControllerTestCase ${CMAKE_SOURCE_DIR}/tests/ControllerTestCase.php ) add_midas_style_test( StyleTestsDatabaseTestCase ${CMAKE_SOURCE_DIR}/tests/DatabaseTestCase.php ) +add_subdirectory( library ) diff --git a/tests/library/CMakeLists.txt b/tests/library/CMakeLists.txt new file mode 100644 index 000000000..36ca5df2b --- /dev/null +++ b/tests/library/CMakeLists.txt @@ -0,0 +1,6 @@ + +add_midas_test( KWUtils KWUtilsTest.php ) + +# Style +add_midas_style_test( StyleKWUtils ${CMAKE_SOURCE_DIR}/library/KWUtils.php ) +add_midas_style_test( StyleKWUtilsTest ${CMAKE_SOURCE_DIR}/tests/library/KWUtilsTest.php ) \ No newline at end of file diff --git a/tests/library/KWUtilsTest.php b/tests/library/KWUtilsTest.php new file mode 100644 index 000000000..cdeba612e --- /dev/null +++ b/tests/library/KWUtilsTest.php @@ -0,0 +1,153 @@ +assertFalse(KWUtils::mkDir($tmpDir)); + $tmpDir .= "/KWUtilsTest"; + $this->assertTrue(KWUtils::mkDir($tmpDir)); + // now clean up + rmdir($tmpDir); + } + + /** tests createSubDirectories function */ + public function testCreateSubDirectories() + { + // test creating directories, do this in the tmp dir + // + // create a nested set of directories + $tmpDir = BASE_PATH . 'tmp/'; + $subDirs = array("KWUtilsTest", "1", "2", "3"); + $outDir = KWUtils::createSubDirectories($tmpDir, $subDirs); + + // now check that all the subdirs have been created + + // according to what we wanted + $this->assertFileExists($tmpDir); + // and what we got back + $this->assertFileExists($outDir); + + $currDir = $tmpDir; + foreach($subDirs as $subdir) + { + $currDir = $currDir . '/' . $subdir; + $this->assertFileExists($currDir); + $this->assertTrue(is_dir($currDir)); + } + + // now walk back up the tree and clean up these tmp dirs + foreach($subDirs as $subdir) + { + // we aren't doing anything with $subdir, just iterating once per subdir + rmdir($currDir); + $parts = explode('/', $currDir); + $parts = array_slice($parts, 0, count($parts)-1); + $currDir = implode('/', $parts); + } + } + + + /** tests exec function */ + public function testExec() + { + // not sure how to test this exactly, for now create a tmp dir, check + // the value of pwd in it + + // create a tmp dir for this test + $execDir = BASE_PATH . 'tmp/KWUtilsTest'; + mkdir($execDir); + $cmd = 'pwd'; + $chdir = $execDir; + KWUtils::exec($cmd, $output, $chdir, $returnVal); + // $output should have one value, the same as execDir + + // yuck, need to do a bit of munging to get around tests/.. in BASE_PATH + $execDir = str_replace('tests/../', '', $execDir); + + $this->assertEquals($execDir, $output[0]); + // return_val should be 0 + $this->assertEquals($returnVal, 0); + // now clean up the tmp dir + rmdir($execDir); + } + + /** tests appendStringIfNot function */ + public function testAppendStringIfNot() + { + // try one that doesn't have the suffix: + $subject = 'blah'; + $ext = '.exe'; + $subject = KWUtils::appendStringIfNot($subject, $ext); + $this->assertEquals($subject, 'blah.exe'); + // now try one that already has the suffix + $subject = 'blah'; + $ext = '.exe'; + $subject = KWUtils::appendStringIfNot($subject, $ext); + $this->assertEquals($subject, 'blah.exe'); + } + + /** tests findApp function */ + public function testFindApp() + { + // first try something that should be in the path, php, and check that it + // is executable + $pathToApp = KWUtils::findApp('php', true); + // now try something that is unlikely to be in the path + try + { + $pathToApp = KWUtils::findApp('php_exe_that_is_vanishingly_likley_to_be_in_the_path', true); + $this->fail('Should have caught exception but did not, testFindApp'); + } + catch(Zend_Exception $ze) + { + // if we end up here, that is the correct behavior + $this->assertTrue(true); + } + } + + /** tests isExecutable function */ + public function testIsExecutable() + { + // this is tricky to test, as it is hard to make assumptions that hold + // up across platforms + // + // for now assume that 'pwd' will not be found + $this->assertFalse(KWUtils::isExecutable('pwd', false)); + // but 'pwd' will be found in the path + $this->assertTrue(KWUtils::isExecutable('pwd', true)); + } + + /** tests prepareExecCommand function */ + public function testPrepareExecCommand() + { + $returnVal = KWUtils::prepareExecCommand('php', array('blah1', 'blah2', 'blah3')); + $appPath = KWUtils::findApp('php', true); + $this->assertEquals($returnVal, "'".$appPath."' 'blah1' 'blah2' 'blah3'"); + } + + + + } \ No newline at end of file