diff --git a/recipes/MADL-CPU-OpenMPI/Data-Shredding/DataShredding.py b/recipes/MADL-CPU-OpenMPI/Data-Shredding/DataShredding.py new file mode 100644 index 00000000..0fed8ebc --- /dev/null +++ b/recipes/MADL-CPU-OpenMPI/Data-Shredding/DataShredding.py @@ -0,0 +1,149 @@ +from __future__ import print_function +import json +import os +import sys +import tempfile +from azure.storage.blob import BlockBlobService +from os import listdir +from os.path import isfile, join + +# make sure config data is encode it correctly +def encode(value): + if isinstance(value, type('str')): + return value + return value.encode('utf-8') + +# configuration class +class Configuration: + + def __init__(self, file_name): + if not os.path.exists(file_name): + raise ValueError('Cannot find configuration file "{0}"'. + format(file_name)) + + with open(file_name, 'r') as f: + conf = json.load(f) + + try: + self.node_count = encode(conf['node_count']) + self.thread_count = encode(conf['thread_count']) + self.training_data_shred_count = encode(conf['training_data_shred_count']) + self.dataset_local_directory = encode(conf['dataset_local_directory']) + self.shredded_dataset_local_directory = encode(conf['shredded_dataset_local_directory']) + self.shredded_dataset_Per_Node = encode(conf['shredded_dataset_Per_Node']) + self.container_name = encode(conf['container_name']) + self.trainind_dataset_name = encode(conf['trainind_dataset_name']) + self.training_data_container_name = encode(conf['training_data_container_name']) + self.subscription_id = encode(conf['subscription_id']) + self.secret_key = encode(conf['secret_key']) + self.resource_group = encode(conf['resource_group']) + self.storage_account_name = encode(conf['storage_account']['name']) + self.storage_account_key = encode(conf['storage_account']['key']) + except KeyError as err: + raise AttributeError('Please provide a value for "{0}" configuration key'.format(err.args[0])) + +# load the configuration data +cfg = Configuration('configuration.json') + +# azure block service object +blob_service = BlockBlobService(cfg.storage_account_name, cfg.storage_account_key) + +# container name +azure_blob_container_name = cfg.container_name + +# training data container name +azure_blob_training_data_container_name = cfg.training_data_container_name + +# create the container that will host the data blobs +blob_service.create_container(azure_blob_container_name, fail_on_exist=False) + + +# the function that load the data from the training blob, partition the data +# and upload it to the container blobs +def partition_and_upload_dataset_to_blob(blob_service, azure_blob_container_name): + + # List the blobs in a training container + blobs = [] + marker = None + blobs_size = 1 + while True: + batch = blob_service.list_blobs(azure_blob_training_data_container_name, marker=marker) + blobs.extend(batch) + if not batch.next_marker: + break + marker = batch.next_marker + for blob in blobs: + blobs_size += blob.properties.content_length + print(blob.name) + + # the vm / thread count + vm_thread_count = (int(cfg.node_count) - 1) * int(cfg.thread_count) + + # the file count per vm + file_count = int(cfg.training_data_shred_count) // vm_thread_count + + # the file size + file_size = blobs_size // int(cfg.training_data_shred_count) + + # data path directory + dataset_local_directory = os.path.normpath(cfg.dataset_local_directory) + + #local shredded data directory + shredded_dataset_local_directory = os.path.normpath(cfg.shredded_dataset_local_directory) + shredded_dataset_Per_Node= os.path.normpath(cfg.shredded_dataset_Per_Node) + + # download data from training blob, slice it + print('downloading dataset from blob and create them localy...') + i = 0 + for itr in range(len(blobs)): + blob = blobs[itr] + blob_service.get_blob_to_path(azure_blob_training_data_container_name, + blob.name, os.path.join(dataset_local_directory, blob.name)) + file_name_no_extension, file_extension = os.path.splitext(blob.name) + + lines_bytes_size = 0 + alist = [] + with open(os.path.join(dataset_local_directory, blob.name), 'r') as in_file: + for line in in_file: + lines_bytes_size += sys.getsizeof(line) + alist.append(line) + if(lines_bytes_size >= file_size): + with open(os.path.join(shredded_dataset_local_directory, + file_name_no_extension + '_' + str(itr) + '_' + str(i) + file_extension), 'w') as wr: + for item in alist: + wr.write(item) + lines_bytes_size = 0 + alist = [] + i +=1 + + # combine shreded files into a one file per node + alldatafiles = [f for f in listdir(shredded_dataset_local_directory) if isfile(join(shredded_dataset_local_directory, f))] + low_index = 0 + high_index = file_count + filename = "data.lst" + for vm_count in range(vm_thread_count): + blob_name = cfg.trainind_dataset_name + "-" + "%05d" % (vm_count,) + if(high_index > len(alldatafiles)): + high_index = len(alldatafiles) + if not os.path.exists(os.path.join(shredded_dataset_Per_Node, blob_name)): + os.makedirs(os.path.join(shredded_dataset_Per_Node, blob_name)) + with open(os.path.join(shredded_dataset_Per_Node, blob_name + '\\' + filename), 'w') as outfile: + for itr in range(low_index, high_index): + with open(os.path.join(shredded_dataset_local_directory, alldatafiles[itr])) as infile: + for line in infile: + outfile.write(line) + low_index += file_count + high_index += file_count + + # upload combined sliced data to blobs + for subdir, dirs, files in os.walk(shredded_dataset_Per_Node): + for file in files: + print ( os.path.basename(subdir)) + print (os.path.join(subdir, file)) + blob_service.create_blob_from_path(azure_blob_container_name, os.path.basename(subdir) + '/' + file, os.path.join(subdir, file)) + + + print('Done') + +# begin loading, partitioning and deploying training data +partition_and_upload_dataset_to_blob(blob_service, azure_blob_container_name) diff --git a/recipes/MADL-CPU-OpenMPI/Data-Shredding/README.md b/recipes/MADL-CPU-OpenMPI/Data-Shredding/README.md new file mode 100644 index 00000000..6b3bb9da --- /dev/null +++ b/recipes/MADL-CPU-OpenMPI/Data-Shredding/README.md @@ -0,0 +1,26 @@ +## MADL-CPU-OpenMPI Data Shredding +This Data Shredding recipe shows how to shred and deploy your training data prior to running a training job on Azure VMs via Open MPI. + +### Data Shredding Configuration +Rename the configuration-template.json to configuration.json. The configuration should enable the following properties: +* `node_count` should be set to the number of VMs in the compute pool. +* `thread_count` thread's count per VM. +* `training_data_shred_count` It's advisable to set this number high. This way you only do this step once, and use it for different VMs configuration. +* 'dataset_local_directory' A local directory to download and shred the training data according to 'training_data_shred_count'. +* 'shredded_dataset_Per_Node' A local directory to hold the final data shreds before deploying them to Azure blobs. +* 'container_name' container name where the sliced data will be stored. +* 'trainind_dataset_name' name for the dataset. Used when creating the data blobs. +* 'subscription_id' Azure subscription id. +* 'secret_key' Azure password. +* 'resource_group' Resource group name. +* 'storage_account' storage account name and access key. +* 'training_data_container_name' Container name where the training data is hosted. +*'' + +You can use your own access mechanism (password, access key, etc.). The above is only a one example. Although, make sure to update the python script +every time you make a configuration change. + +You must agree to the following licenses prior to use: +* [High Performance ML Algorithms License](https://github.com/saeedmaleki/Distributed-Linear-Learner/blob/master/High%20Performance%20ML%20Algorithms%20-%20Standalone%20(free)%20Use%20Terms%20V2%20(06-06-18).txt) +* [TPN Ubuntu Container](https://github.com/saeedmaleki/Distributed-Linear-Learner/blob/master/TPN_Ubuntu%20Container_16-04-FINAL.txt) +* [Microsoft Third Party Notice](https://github.com/saeedmaleki/Distributed-Linear-Learner/blob/master/MicrosoftThirdPartyNotice.txt) diff --git a/recipes/MADL-CPU-OpenMPI/Data-Shredding/configuration-template.json b/recipes/MADL-CPU-OpenMPI/Data-Shredding/configuration-template.json new file mode 100644 index 00000000..040b11f8 --- /dev/null +++ b/recipes/MADL-CPU-OpenMPI/Data-Shredding/configuration-template.json @@ -0,0 +1,18 @@ +{ + "node_count": + "thread_count": , + "training_data_shred_count": , + "dataset_local_directory": , + "shredded_dataset_local_directory": , + "shredded_dataset_Per_Node": + "container_name": , + "trainind_dataset_name": , + "secret_key": , + "resource_group": , + "storage_account": { + "name": , + "key": + }, + "training_data_container_name": +} diff --git a/recipes/MADL-CPU-OpenMPI/README.md b/recipes/MADL-CPU-OpenMPI/README.md new file mode 100644 index 00000000..5cd79a0c --- /dev/null +++ b/recipes/MADL-CPU-OpenMPI/README.md @@ -0,0 +1,60 @@ +# MADL-CPU-OpenMPI +This recipe shows how to run High Performance ML Algorithms Learner on CPUs across +Azure VMs via Open MPI. + +## Configuration +Please see refer to this [set of sample configuration files](./config) for +this recipe. + +### Pool Configuration +The pool configuration should enable the following properties: +* `vm_size` should be a CPU-only instance, for example, 'STANDARD_D2_V2'. +* `inter_node_communication_enabled` must be set to `true` +* `max_tasks_per_node` must be set to 1 or omitted + +### Global Configuration +The global configuration should set the following properties: +* `docker_images` array must have a reference to a valid MADL +Docker image that can be run with OpenMPI. The image denoted with `0.0.1` tag found in [msmadl/symsgd:0.0.1](https://hub.docker.com/r/msmadl/symsgd/) +is compatible with Azure Batch Shipyard VMs. + +### MPI Jobs Configuration (MultiNode) +The jobs configuration should set the following properties within the `tasks` +array which should have a task definition containing: +* `docker_image` should be the name of the Docker image for this container invocation. +For this example, this should be +`msmadl/symsgd:0.0.1`. +Please note that the `docker_images` in the Global Configuration should match +this image name. +* `command` should contain the command to pass to the Docker run invocation. +For this MADL training example with the `msmadl/symsgd:0.0.1` Docker image. The +application `command` to run would be: +`"/parasail/run_parasail.sh -w /parasail/supersgd -l 1e-4 -k 32 -m 1e-2 -e 10 -r 10 -f $AZ_BATCH_NODE_SHARED_DIR/azblob/ -t 1 -g 1 -d $AZ_BATCH_TASK_WORKING_DIR/models -b $AZ_BATCH_NODE_SHARED_DIR/azblob/"` + * [`run_parasail.sh`](docker/run_parasail.sh) has these parameters + * `-w` the MADL superSGD directory + * `-l` learning rate + * `-k` approximation rank constant + * `-m` model combiner convergence threshold + * `-e` total epochs + * `-r` rounds per epoch + * `-f` training file prefix + * `-t` number of threads + * `-g` log global models every this many epochs + * `-d` log global models to this directory at the host" + * `-b` location for the algorithm's binary" + +* The training data will need to be shredded to match the number of VMs and the thread's count per VM, and then deployed to a mounted Azure blob that the VM docker images have read/write access. +A basic python script that can be used to shred and deploy the training data to a blob container, and other data shredding files can be found [here](./DataShredding). +* `shared_data_volumes` should contain the shared data volume with an `azureblob` volume driver as specified in the global configuration file found [here](./config/config.yaml). + +* `multi_instance` property must be defined + * `num_instances` should be set to `pool_current_dedicated`, or + `pool_current_low_priority` + +## Dockerfile and supplementary files +Supplementary files can be found [here](./docker). + +You must agree to the following licenses prior to use: +* [High Performance ML Algorithms License](https://github.com/saeedmaleki/Distributed-Linear-Learner/blob/master/High%20Performance%20ML%20Algorithms%20-%20Standalone%20(free)%20Use%20Terms%20V2%20(06-06-18).txt) +* [TPN Ubuntu Container](https://github.com/saeedmaleki/Distributed-Linear-Learner/blob/master/TPN_Ubuntu%20Container_16-04-FINAL.txt) +* [Microsoft Third Party Notice](https://github.com/saeedmaleki/Distributed-Linear-Learner/blob/master/MicrosoftThirdPartyNotice.txt) diff --git a/recipes/MADL-CPU-OpenMPI/config/config.yaml b/recipes/MADL-CPU-OpenMPI/config/config.yaml new file mode 100644 index 00000000..1f2d6b19 --- /dev/null +++ b/recipes/MADL-CPU-OpenMPI/config/config.yaml @@ -0,0 +1,13 @@ +batch_shipyard: + storage_account_settings: mystorageaccount +global_resources: + docker_images: + - msmadl/symsgd:0.0.1 + volumes: + shared_data_volumes: + azureblob_vol: + volume_driver: azureblob + storage_account_settings: mystorageaccount + azure_blob_container_name: + container_path: $AZ_BATCH_NODE_SHARED_DIR/azblob + bind_options: rw diff --git a/recipes/MADL-CPU-OpenMPI/config/credentials.yaml b/recipes/MADL-CPU-OpenMPI/config/credentials.yaml new file mode 100644 index 00000000..797a2511 --- /dev/null +++ b/recipes/MADL-CPU-OpenMPI/config/credentials.yaml @@ -0,0 +1,9 @@ +credentials: + batch: + account_key: + account_service_url: + storage: + mystorageaccount: + account: + account_key: + endpoint: core.windows.net \ No newline at end of file diff --git a/recipes/MADL-CPU-OpenMPI/config/jobs.yaml b/recipes/MADL-CPU-OpenMPI/config/jobs.yaml new file mode 100644 index 00000000..44033591 --- /dev/null +++ b/recipes/MADL-CPU-OpenMPI/config/jobs.yaml @@ -0,0 +1,10 @@ +job_specifications: +- id: + auto_complete: true + shared_data_volumes: + - azureblob_vol + tasks: + - docker_image: msmadl/symsgd:0.0.1 + multi_instance: + num_instances: pool_current_dedicated + command: /parasail/run_parasail.sh -w /parasail/supersgd -l 1e-4 -k 32 -m 1e-2 -e 10 -r 10 -f $AZ_BATCH_NODE_SHARED_DIR/azblob/ -t 1 -g 1 -d $AZ_BATCH_TASK_WORKING_DIR/models -b $AZ_BATCH_NODE_SHARED_DIR/azblob/ diff --git a/recipes/MADL-CPU-OpenMPI/config/pool.yaml b/recipes/MADL-CPU-OpenMPI/config/pool.yaml new file mode 100644 index 00000000..13db659d --- /dev/null +++ b/recipes/MADL-CPU-OpenMPI/config/pool.yaml @@ -0,0 +1,12 @@ +pool_specification: + id: + vm_configuration: + platform_image: + offer: UbuntuServer + publisher: Canonical + sku: 16.04-LTS + vm_count: + dedicated: 3 + low_priority: 0 + vm_size: STANDARD_D2_V2 + inter_node_communication_enabled: true diff --git a/recipes/MADL-CPU-OpenMPI/docker/Dockerfile b/recipes/MADL-CPU-OpenMPI/docker/Dockerfile new file mode 100644 index 00000000..a3819e51 --- /dev/null +++ b/recipes/MADL-CPU-OpenMPI/docker/Dockerfile @@ -0,0 +1,45 @@ +#Dockerfile for MADL (Microsoft Distributed Learners) + +FROM ubuntu:16.04 +MAINTAINER Saeed Maleki Todd Mytkowicz Madan Musuvathi Dany rouhana https://github.com/saeedmaleki/Distributed-Linear-Learner +ENV DEBIAN_FRONTEND=noninteractive + +#install base system +RUN apt-get update && apt-get install -y --no-install-recommends \ + openssh-client \ + openssh-server \ + libopenblas-dev \ + libatlas-base-dev \ + liblapacke-dev \ + openmpi-bin \ + openmpi-common \ + libopenmpi-dev && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# configure ssh server and keys +RUN mkdir -p /root/.ssh && \ + echo "Host *\n\tPort 23\n\tStrictHostKeyChecking no\n\tUserKnownHostsFile /dev/null" > /root/.ssh/config && \ + mkdir /var/run/sshd && \ + ssh-keygen -A && \ + sed -i 's/PermitRootLogin without-password/PermitRootLogin yes/' /etc/ssh/sshd_config && \ + sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd && \ + ssh-keygen -f /root/.ssh/id_rsa -t rsa -N '' && \ + chmod 600 /root/.ssh/config && \ + chmod 700 /root/.ssh && \ + cp /root/.ssh/id_rsa.pub /root/.ssh/authorized_keys + +# set parasail dir +WORKDIR /parasail + +# to create your own image, first download the supersgd from the link supplied in the read me file, +# and the put it in the same dir as this file. +COPY supersgd /parasail +COPY run_parasail.sh /parasail + +# remove romio314 bits +RUN rm -rf /usr/lib/openmpi/lib/openmpi/mca_io_romio.so + +#make sshd listen on 23 and run by default +EXPOSE 23 +CMD ["/usr/sbin/sshd", "-D", "-p", "23"] diff --git a/recipes/MADL-CPU-OpenMPI/docker/README.md b/recipes/MADL-CPU-OpenMPI/docker/README.md new file mode 100644 index 00000000..0799f9eb --- /dev/null +++ b/recipes/MADL-CPU-OpenMPI/docker/README.md @@ -0,0 +1,9 @@ +# Dockerfile for msmadl/symsgd +This image can be found in [Docker Hub](https://hub.docker.com/r/msmadl/symsgd/) + +You can use our algorithm by downloading our docker image (docker pull msmadl/symsgd). If you decide to build your own, you first need to download the High Performance ML Algorithms Learner binary from this [link](https://github.com/saeedmaleki/Distributed-Linear-Learner/blob/master/supersgd), and include it in the same directory as your Dockerfile. + +You must agree to the following licenses prior to use: +* [High Performance ML Algorithms License](https://github.com/saeedmaleki/Distributed-Linear-Learner/blob/master/High%20Performance%20ML%20Algorithms%20-%20Standalone%20(free)%20Use%20Terms%20V2%20(06-06-18).txt) +* [TPN Ubuntu Container](https://github.com/saeedmaleki/Distributed-Linear-Learner/blob/master/TPN_Ubuntu%20Container_16-04-FINAL.txt) +* [Microsoft Third Party Notice](https://github.com/saeedmaleki/Distributed-Linear-Learner/blob/master/MicrosoftThirdPartyNotice.txt) diff --git a/recipes/MADL-CPU-OpenMPI/docker/run_parasail.sh b/recipes/MADL-CPU-OpenMPI/docker/run_parasail.sh new file mode 100644 index 00000000..ead81a8f --- /dev/null +++ b/recipes/MADL-CPU-OpenMPI/docker/run_parasail.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash + +# get VMs Nodes +IFS=',' read -ra HOSTS <<< "$AZ_BATCH_HOST_LIST" +nodes=${#HOSTS[@]} +# print configuration +echo num nodes: $nodes +echo "hosts: ${HOSTS[@]}" + +# create output directories +mkdir $AZ_BATCH_TASK_WORKING_DIR/models + +echo "set variables" +set -e +w= +l= +k= +mc= +e= +r= +f= +t= +gl= +glDir= +bd= +while getopts "h?w:l:k:m:e:r:f:t:n:g:d:b:" opt; do + case "$opt" in + h|\?) + echo "run_parasail.sh parameters" + echo "" + echo "-w superSGD directory" + echo "-l learning rate" + echo "-k approximation rank constant" + echo "-mc model combiner convergence threshold" + echo "-e total epochs" + echo "-r rounds per epoch" + echo "-f file prefix" + echo "-t number of threads" + echo "-gl log global models every this many epochs" + echo "-glDir log global models to this directory at the host" + echo "-bd location for the binary data" + echo "" + exit 1 + ;; + w) + w=${OPTARG} + ;; + l) + l=${OPTARG} + ;; + k) + k=${OPTARG} + ;; + m) + mc=${OPTARG} + ;; + e) + e=${OPTARG} + ;; + r) + r=${OPTARG} + ;; + f) + f=${OPTARG} + ;; + t) + t=${OPTARG} + ;; + g) + gl=${OPTARG} + ;; + d) + glDir=${OPTARG} + ;; + b) + bd=${OPTARG} + ;; + esac +done +shift $((OPTIND-1)) +[ "$1" = "--" ] && shift +echo "end set variables" + +echo "start mpi execute job" +mpirun --allow-run-as-root --mca btl_tcp_if_exclude docker0 -np $nodes $w -l $l -k $k -mc $mc -e $e -r $r -f $f -t $t -gl $gl -glDir $glDir -mem -bd $bd -dl +echo "end mpi job" diff --git a/recipes/MADL-CPU-OpenMPI/docker/ssh_config b/recipes/MADL-CPU-OpenMPI/docker/ssh_config new file mode 100644 index 00000000..f366599c --- /dev/null +++ b/recipes/MADL-CPU-OpenMPI/docker/ssh_config @@ -0,0 +1,4 @@ +Host 10.* + Port 23 + StrictHostKeyChecking no + UserKnownHostsFile /dev/null \ No newline at end of file