-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #2 from mathworks/discovery
Add cluster discovery files
- Loading branch information
Showing
10 changed files
with
298 additions
and
86 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,30 @@ | ||
| function createEnvironmentWrapper(outputFilename, quotedWrapperPath, environmentVariables) | ||
| % Create a script that sets the correct environment variables and then | ||
| % calls the job wrapper. | ||
|
|
||
| % Copyright 2023 The MathWorks, Inc. | ||
|
|
||
| dctSchedulerMessage(5, '%s: Creating environment wrapper at %s', mfilename, outputFilename); | ||
|
|
||
| % Open file in binary mode to make it cross-platform. | ||
| fid = fopen(outputFilename, 'w'); | ||
| if fid < 0 | ||
| error('parallelexamples:GenericSLURM:FileError', ... | ||
| 'Failed to open file %s for writing', outputFilename); | ||
| end | ||
| fileCloser = onCleanup(@() fclose(fid)); | ||
|
|
||
| % Specify shell to use | ||
| fprintf(fid, '#!/bin/sh\n'); | ||
|
|
||
| formatSpec = 'export %s=''%s''\n'; | ||
|
|
||
| % Write the commands to set and export environment variables | ||
| for ii = 1:size(environmentVariables, 1) | ||
| fprintf(fid, formatSpec, environmentVariables{ii,1}, environmentVariables{ii,2}); | ||
| end | ||
|
|
||
| % Write the command to run the job wrapper | ||
| fprintf(fid, '%s\n', quotedWrapperPath); | ||
|
|
||
| end |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,85 @@ | ||
| # Since version R2023a, MATLAB can discover clusters running third-party | ||
| # schedulers such as Slurm. The Discover Clusters functionality | ||
| # automatically configures the Parallel Computing Toolbox to submit MATLAB | ||
| # jobs to the cluster. To use this functionality, you must create a cluster | ||
| # configuration file and store it at a location accessible to MATLAB users. | ||
| # | ||
| # This file is an example of a cluster configuration which MATLAB can | ||
| # discover. You can copy and modify this file to make your cluster discoverable. | ||
| # | ||
| # For more information, including the required format for this file, see | ||
| # the online documentation for making a cluster running a third-party | ||
| # scheduler discoverable: | ||
| # https://mathworks.com/help/matlab-parallel-server/configure-for-cluster-discovery.html | ||
|
|
||
| # Copyright 2023 The MathWorks, Inc. | ||
|
|
||
| # The name MATLAB will display for the cluster when discovered. | ||
| Name = My Slurm cluster | ||
|
|
||
| # Maximum number of MATLAB workers a single user can use in a single job. | ||
| # This number must not exceed the number of available MATLAB Parallel | ||
| # Server licenses. | ||
| NumWorkers = 32 | ||
|
|
||
| # Path to the MATLAB install on the cluster for the workers to use. Note | ||
| # the variable "$MATLAB_VERSION_STRING" returns the release number of the | ||
| # MATLAB client that is running discovery, e.g. 2023a. If multiple versions | ||
| # of MATLAB are installed on the cluster, this allows discovery to select | ||
| # the correct installation path. Add a leading "R" or "r" if needed to | ||
| # complete the MATLAB version. | ||
| ClusterMatlabRoot = /opt/matlab/R"$MATLAB_VERSION_STRING" | ||
|
|
||
| # Location where the MATLAB client stores job and task information. | ||
| JobStorageLocation = /home/matlabjobs | ||
| # If the client and cluster share a filesystem but the client is running | ||
| # the Windows operating system and the cluster running a Linux operating | ||
| # system, you must specify the JobStorageLocation using a structure by | ||
| # commenting out the previous line and uncommenting the following lines. | ||
| # The 'windows' and 'unix' fields must correspond to the same folder as | ||
| # viewed from each of those operating systems. | ||
| #JobStorageLocation.windows = \\organization\home\matlabjobs | ||
| #JobStorageLocation.unix = /organization/home/matlabjobs | ||
|
|
||
| # Folder that contains the scheduler plugin scripts that describe how | ||
| # MATLAB interacts with the scheduler. A property can take different values | ||
| # depending on the operating system of the client MATLAB by specifying the | ||
| # name of the OS in parentheses. | ||
| PluginScriptsLocation (Windows) = \\organization\matlab\pluginscripts | ||
| PluginScriptsLocation (Unix) = /organization/matlab/pluginscripts | ||
|
|
||
| # The operating system on the cluster. Valid values are 'unix' and 'windows'. | ||
| OperatingSystem = unix | ||
|
|
||
| # Specify whether client and cluster nodes share JobStorageLocation. To | ||
| # configure MATLAB to copy job input and output files to and from the | ||
| # cluster using SFTP, set this property to false and specify a value for | ||
| # AdditionalProperties.RemoteJobStorageLocation below. | ||
| HasSharedFilesystem = true | ||
|
|
||
| # Specify whether the cluster uses online licensing. | ||
| RequiresOnlineLicensing = false | ||
|
|
||
| # LicenseNumber for the workers to use. Specify only if | ||
| # RequiresOnlineLicensing is set to true. | ||
| #LicenseNumber = 123456 | ||
|
|
||
| [AdditionalProperties] | ||
|
|
||
| # To configure the user's machine to connect to the submission host via | ||
| # SSH, uncomment the following line and enter the hostname of the cluster | ||
| # machine that has the scheduler utilities to submit jobs. | ||
| #ClusterHost = slurm-headnode | ||
|
|
||
| # If the user's machine and the cluster nodes do not have a shared file | ||
| # system, MATLAB can copy job input and output files to and from the | ||
| # cluster using SFTP. To activate this feature, set HasSharedFilesystem | ||
| # above to false. Then uncomment the following lines and enter the location | ||
| # on the cluster to store job files. | ||
| #RemoteJobStorageLocation (Windows) = /home/"$USERNAME"/.matlab/generic_cluster_jobs | ||
| #RemoteJobStorageLocation (Unix) = /home/"$USER"/.matlab/generic_cluster_jobs | ||
|
|
||
| # Username to log in to ClusterHost with. On Linux and Mac, use the USER | ||
| # environment variable. On Windows, use the USERNAME variable. | ||
| Username (Unix) = "$USER" | ||
| Username (Windows) = "$USERNAME" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,37 @@ | ||
| #!/bin/sh | ||
|
|
||
| # Copyright 2023 The MathWorks, Inc. | ||
|
|
||
| usage="$(basename "$0") matlabroot [folder] -- run third-party scheduler discovery in MATLAB R2023a onwards | ||
| matlabroot - path to the folder where MATLAB is installed | ||
| folder - folder to search for cluster configuration files | ||
| (defaults to pwd)" | ||
|
|
||
| # Print usage | ||
| if [ -z "$1" ] || [ "$1" = "-h" ] || [ "$1" = "--help" ] ; then | ||
| echo "$usage" | ||
| exit 0 | ||
| fi | ||
|
|
||
| # MATLAB executable to launch | ||
| matlabExe="$1/bin/matlab" | ||
| if [ ! -f "${matlabExe}" ] ; then | ||
| echo "Could not find MATLAB executable at ${matlabExe}" | ||
| exit 1 | ||
| fi | ||
|
|
||
| # Folder to run discovery on. If specified, wrap in single-quotes to make a MATLAB charvec. | ||
| discoveryFolder="$2" | ||
| if [ ! -z "$discoveryFolder" ] ; then | ||
| discoveryFolder="'${discoveryFolder}'" | ||
| fi | ||
|
|
||
| # Command to run in MATLAB | ||
| matlabCmd="parallel.cluster.generic.discoverGenericClusters(${discoveryFolder})" | ||
|
|
||
| # Arguments to pass to MATLAB | ||
| matlabArgs="-nojvm -parallelserver -batch" | ||
|
|
||
| # Build and run system command | ||
| CMD="\"${matlabExe}\" ${matlabArgs} \"${matlabCmd}\"" | ||
| eval $CMD |
Oops, something went wrong.