slurm_tutorial.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Slurm Tutorial\n",
    " - sinfo\n",
    " - squeue\n",
    " - scancel\n",
    " - salloc\n",
    " - srun"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# sinfo provides information on the current cluster\n",
    "\n",
    "## sinfo help"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Usage: sinfo [OPTIONS]\n",
      "  -a, --all                  show all partitions (including hidden and those\n",
      "\t\t\t     not accessible)\n",
      "  -d, --dead                 show only non-responding nodes\n",
      "  -e, --exact                group nodes only on exact match of configuration\n",
      "      --federation           Report federated information if a member of one\n",
      "  -h, --noheader             no headers on output\n",
      "  --hide                     do not show hidden or non-accessible partitions\n",
      "  -i, --iterate=seconds      specify an iteration period\n",
      "      --local                show only local cluster in a federation.\n",
      "                             Overrides --federation.\n",
      "  -l, --long                 long output - displays more information\n",
      "  -M, --clusters=names       clusters to issue commands to. Implies --local.\n",
      "                             NOTE: SlurmDBD must be up.\n",
      "  -n, --nodes=NODES          report on specific node(s)\n",
      "  --noconvert                don't convert units from their original type\n",
      "\t\t\t     (e.g. 2048M won't be converted to 2G).\n",
      "  -N, --Node                 Node-centric format\n",
      "  -o, --format=format        format specification\n",
      "  -O, --Format=format        long format specification\n",
      "  -p, --partition=PARTITION  report on specific partition\n",
      "  -r, --responding           report only responding nodes\n",
      "  -R, --list-reasons         list reason nodes are down or drained\n",
      "  -s, --summarize            report state summary only\n",
      "  -S, --sort=fields          comma separated list of fields to sort on\n",
      "  -t, --states=node_state    specify the what states of nodes to view\n",
      "  -T, --reservation          show only reservation information\n",
      "  -v, --verbose              verbosity level\n",
      "  -V, --version              output version information and exit\n",
      "\n",
      "Help options:\n",
      "  --help                     show this help message\n",
      "  --usage                    display brief usage message\n"
     ]
    }
   ],
   "source": [
    "!sinfo --help"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PARTITION AVAIL  TIMELIMIT  NODES  STATE NODELIST\n",
      "compute*     up   infinite      4   idle ip-172-31-37-10,ip-172-31-46-[20,159],ip-172-31-47-236\n"
     ]
    }
   ],
   "source": [
    "!sinfo -a"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get Sockets, Memory, Cores information on each compute node"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SOCKETS\n",
      "4\n"
     ]
    }
   ],
   "source": [
    "!sinfo -o %X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CORES\n",
      "1\n"
     ]
    }
   ],
   "source": [
    "!sinfo -o %Y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "THREADS\n",
      "1\n"
     ]
    }
   ],
   "source": [
    "!sinfo -o %Z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "FREE_MEM\n",
      "15081-15082\n"
     ]
    }
   ],
   "source": [
    "!sinfo -o %e"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "!sinfo -o %all > sinfo_all.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>AVAIL</th>\n",
       "      <th>ACTIVE_FEATURES</th>\n",
       "      <th>CPUS</th>\n",
       "      <th>TMP_DISK</th>\n",
       "      <th>FREE_MEM</th>\n",
       "      <th>AVAIL_FEATURES</th>\n",
       "      <th>GROUPS</th>\n",
       "      <th>OVERSUBSCRIBE</th>\n",
       "      <th>TIMELIMIT</th>\n",
       "      <th>MEMORY</th>\n",
       "      <th>...</th>\n",
       "      <th>CPU_LOAD</th>\n",
       "      <th>PARTITION</th>\n",
       "      <th>PARTITION .1</th>\n",
       "      <th>ALLOCNODES</th>\n",
       "      <th>STATE</th>\n",
       "      <th>USER</th>\n",
       "      <th>CLUSTER</th>\n",
       "      <th>SOCKETS</th>\n",
       "      <th>CORES</th>\n",
       "      <th>THREADS</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>up</td>\n",
       "      <td>(null)</td>\n",
       "      <td>4</td>\n",
       "      <td>49569</td>\n",
       "      <td>15082</td>\n",
       "      <td>(null)</td>\n",
       "      <td>all</td>\n",
       "      <td>NO</td>\n",
       "      <td>infinite</td>\n",
       "      <td>16046</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>compute*</td>\n",
       "      <td>compute</td>\n",
       "      <td>all</td>\n",
       "      <td>idle</td>\n",
       "      <td>root(0)</td>\n",
       "      <td>N/A</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>up</td>\n",
       "      <td>(null)</td>\n",
       "      <td>4</td>\n",
       "      <td>49569</td>\n",
       "      <td>15081</td>\n",
       "      <td>(null)</td>\n",
       "      <td>all</td>\n",
       "      <td>NO</td>\n",
       "      <td>infinite</td>\n",
       "      <td>16046</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>compute*</td>\n",
       "      <td>compute</td>\n",
       "      <td>all</td>\n",
       "      <td>idle</td>\n",
       "      <td>root(0)</td>\n",
       "      <td>N/A</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>up</td>\n",
       "      <td>(null)</td>\n",
       "      <td>4</td>\n",
       "      <td>49569</td>\n",
       "      <td>15081</td>\n",
       "      <td>(null)</td>\n",
       "      <td>all</td>\n",
       "      <td>NO</td>\n",
       "      <td>infinite</td>\n",
       "      <td>16046</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>compute*</td>\n",
       "      <td>compute</td>\n",
       "      <td>all</td>\n",
       "      <td>idle</td>\n",
       "      <td>root(0)</td>\n",
       "      <td>N/A</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>up</td>\n",
       "      <td>(null)</td>\n",
       "      <td>4</td>\n",
       "      <td>49569</td>\n",
       "      <td>15082</td>\n",
       "      <td>(null)</td>\n",
       "      <td>all</td>\n",
       "      <td>NO</td>\n",
       "      <td>infinite</td>\n",
       "      <td>16046</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>compute*</td>\n",
       "      <td>compute</td>\n",
       "      <td>all</td>\n",
       "      <td>idle</td>\n",
       "      <td>root(0)</td>\n",
       "      <td>N/A</td>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4 rows × 42 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  AVAIL ACTIVE_FEATURES  CPUS  TMP_DISK  FREE_MEM AVAIL_FEATURES GROUPS  \\\n",
       "0    up          (null)     4     49569     15082         (null)    all   \n",
       "1    up          (null)     4     49569     15081         (null)    all   \n",
       "2    up          (null)     4     49569     15081         (null)    all   \n",
       "3    up          (null)     4     49569     15082         (null)    all   \n",
       "\n",
       "  OVERSUBSCRIBE TIMELIMIT  MEMORY  ... CPU_LOAD  PARTITION   PARTITION .1  \\\n",
       "0            NO  infinite   16046  ...       0.0  compute*       compute    \n",
       "1            NO  infinite   16046  ...       0.0  compute*       compute    \n",
       "2            NO  infinite   16046  ...       0.0  compute*       compute    \n",
       "3            NO  infinite   16046  ...       0.0  compute*       compute    \n",
       "\n",
       "  ALLOCNODES  STATE      USER  CLUSTER   SOCKETS   CORES  THREADS   \n",
       "0        all   idle   root(0)      N/A          4       1        1  \n",
       "1        all   idle   root(0)      N/A          4       1        1  \n",
       "2        all   idle   root(0)      N/A          4       1        1  \n",
       "3        all   idle   root(0)      N/A          4       1        1  \n",
       "\n",
       "[4 rows x 42 columns]"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_table(\"sinfo_all.txt\",delimiter=\"|\")\n",
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             JOBID PARTITION     NAME     USER ST       TIME  NODES NODELIST(REASON)\n"
     ]
    }
   ],
   "source": [
    "!squeue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ip-172-31-37-10\n",
      "ip-172-31-46-20\n"
     ]
    }
   ],
   "source": [
    "!srun -N 2 -n 2 hostname"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "salloc: Granted job allocation 9\n",
      "ip-172-31-33-28\n",
      "salloc: Relinquishing job allocation 9\n"
     ]
    }
   ],
   "source": [
    "!salloc -N 2 -n 2 hostname"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}