demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "## load smartSLURM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from py4j.java_gateway import JavaGateway\n",
    "GATEWAY = JavaGateway.launch_gateway(classpath='include/smartSLURM.jar')\n",
    "ServerSSH = GATEWAY.jvm.com.chanzy.ServerSSH\n",
    "ServerSLURM = GATEWAY.jvm.com.chanzy.ServerSLURM"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "## set login parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "SSH_USER = 'username'\n",
    "SSH_IP = 'hostname'\n",
    "SSH_PORT = 22\n",
    "SSH_PASSWORD = 'password'\n",
    "SERVER_PROJECT_PATH = '~/test-smartSLURM'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "## SSH PART"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import time\n",
    "# init ssh, remote project path is needed\n",
    "ssh = ServerSSH.get(SERVER_PROJECT_PATH, SSH_USER, SSH_IP, SSH_PORT, SSH_PASSWORD)\n",
    "# put all of this project dir to the server (will exclude file and dir which begin with '.' or '_')\n",
    "ssh.putWorkingDir()\n",
    "# can mkdir in ssh\n",
    "ssh.mkdir('.test1/.test2/.test3')\n",
    "# submit command to remote server by using 'system' like matlab\n",
    "ssh.system('ls')\n",
    "ssh.system('echo \"I am using smartSlURM system\" > .test1/.test2/.test3/out.txt')\n",
    "# get dir from remote server by using 'getDir'\n",
    "ssh.getDir('.test1')\n",
    "ssh.getDir('.test1', 4) # using 4 threads\n",
    "# set compression level for large file transmission\n",
    "ssh.setCompressionLevel(4)\n",
    "ssh.putDir('.test1')\n",
    "# unset compression by setting a negative level\n",
    "ssh.setCompressionLevel(-1)\n",
    "# you can get the pool to submit commands parallel\n",
    "pool = ssh.pool(4)\n",
    "pool.submitSystem('sleep 1s')\n",
    "pool.submitSystem('sleep 2s')\n",
    "pool.submitSystem('sleep 3s')\n",
    "pool.submitSystem('sleep 4s')\n",
    "pool.submitSystem('sleep 5s')\n",
    "start = time.time()\n",
    "pool.waitUntilDone()\n",
    "end = time.time()\n",
    "print(end-start)\n",
    "pool.shutdown()\n",
    "# use 'shutdown()' to disconnect at last\n",
    "ssh.shutdown()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "## SLURM PART"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# init slurm, max job number, remote project path is needed\n",
    "slurm = ServerSLURM.get(4, SERVER_PROJECT_PATH, SSH_USER, SSH_IP, SSH_PORT, SSH_PASSWORD)\n",
    "# you can get the ssh of this and use the ssh methods before\n",
    "ssh = slurm.ssh()\n",
    "# use 'submitSystem' to submit command in sbatch mod\n",
    "# slurm.submitSystem(Command, [Partition='work'], NodeNumber=1, OutputPath='.temp/slurm/out-%j');\n",
    "slurm.submitSystem('echo \"I am using smartSlURM submitSystem\"')\n",
    "# use 'submitSrun' to submit a srun command directly (also in sbatch mod)\n",
    "# slurm.submitSrun(Command, [Partition='work'], TaskNumber=1, MaxTaskNumberPerNode=20, OutputPath='.temp/slurm/out-%j');\n",
    "slurm.submitSrun('echo \"I am using smartSlURM submitSrun\"')\n",
    "# use 'submitSrunBash' to submit a bash file and use srun to exec (also in sbatch mod)\n",
    "# slurm.submitSrunBash(BashPath, [Partition='work'], TaskNumber=1, MaxTaskNumberPerNode=20, OutputPath='.temp/slurm/out-%j');\n",
    "slurm.submitSrun('path/to/your/script')\n",
    "# you can use 'undo' to poll the last command in queue list\n",
    "cmd = slurm.undo()\n",
    "# use 'getActiveCount' and 'getQueueSize' to monitor task situation\n",
    "activeNum = slurm.getActiveCount()\n",
    "queueNum = slurm.getQueueSize()\n",
    "taskNum = slurm.getTaskNumber() # getActiveCount() + getQueueSize();\n",
    "# you can use `waitUntilDone` to wait all task complete\n",
    "slurm.waitUntilDone()\n",
    "# or can use the classic way in java thread pool\n",
    "slurm.shutdown()\n",
    "slurm.awaitTermination()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": false
   },
   "source": [
    "## unload smartSLURM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "GATEWAY.shutdown()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}