diff --git a/docs/reference/constants.md b/docs/reference/constants.md new file mode 100644 index 00000000..dd659b4c --- /dev/null +++ b/docs/reference/constants.md @@ -0,0 +1,7 @@ +--- +title: constants +--- + +::: pyslurm.constants + handler: python + diff --git a/docs/reference/old/partition.md b/docs/reference/old/partition.md new file mode 100644 index 00000000..0e69bbfb --- /dev/null +++ b/docs/reference/old/partition.md @@ -0,0 +1,10 @@ +--- +title: Partition +--- + +!!! warning + This class is superseded by [pyslurm.Partition](../partition.md) and will + be removed in a future release. + +::: pyslurm.partition + handler: python diff --git a/docs/reference/partition.md b/docs/reference/partition.md index 6ab4b865..b9701f55 100644 --- a/docs/reference/partition.md +++ b/docs/reference/partition.md @@ -2,9 +2,12 @@ title: Partition --- -!!! warning - This API is currently being completely reworked, and is subject to be - removed in the future when a replacement is introduced +!!! note + This supersedes the [pyslurm.partition](old/partition.md) class, which + will be removed in a future release -::: pyslurm.partition +::: pyslurm.Partition + handler: python + +::: pyslurm.Partitions handler: python diff --git a/docs/reference/utilities.md b/docs/reference/utilities.md index b290d055..63eb7bc0 100644 --- a/docs/reference/utilities.md +++ b/docs/reference/utilities.md @@ -1,5 +1,5 @@ --- -title: Utilities +title: utils --- ::: pyslurm.utils diff --git a/pyslurm/__init__.py b/pyslurm/__init__.py index 750199da..06bd804b 100644 --- a/pyslurm/__init__.py +++ b/pyslurm/__init__.py @@ -14,6 +14,7 @@ from pyslurm import utils from pyslurm import db +from pyslurm import constants from pyslurm.core.job import ( Job, @@ -23,6 +24,7 @@ JobSubmitDescription, ) from pyslurm.core.node import Node, Nodes +from pyslurm.core.partition import Partition, Partitions from pyslurm.core import error from pyslurm.core.error import ( PyslurmError, diff --git a/pyslurm/constants.py b/pyslurm/constants.py new file mode 100644 index 00000000..0b3c11b0 --- /dev/null +++ b/pyslurm/constants.py @@ -0,0 +1,32 @@ +######################################################################### +# constants.py - pyslurm constants used throughout the project +######################################################################### +# Copyright (C) 2023 Toni Harzendorf +# Copyright (C) 2023 PySlurm Developers +# +# This file is part of PySlurm +# +# PySlurm is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# PySlurm is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with PySlurm; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# cython: c_string_type=unicode, c_string_encoding=default +# cython: language_level=3 +"""pyslurm common Constants""" + + +UNLIMITED = "UNLIMITED" +""" +Represents an infinite/unlimited value. This is sometimes returned for +specific attributes as a value to indicate that there is no restriction for it. +""" diff --git a/pyslurm/core/partition.pxd b/pyslurm/core/partition.pxd new file mode 100644 index 00000000..9baeba62 --- /dev/null +++ b/pyslurm/core/partition.pxd @@ -0,0 +1,223 @@ +######################################################################### +# partition.pxd - interface to work with partitions in slurm +######################################################################### +# Copyright (C) 2023 Toni Harzendorf +# Copyright (C) 2023 PySlurm Developers +# +# This file is part of PySlurm +# +# PySlurm is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# PySlurm is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with PySlurm; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# cython: c_string_type=unicode, c_string_encoding=default +# cython: language_level=3 + +from libc.string cimport memcpy, memset +from pyslurm cimport slurm +from libc.stdint cimport uint8_t, uint16_t, uint32_t, uint64_t +from pyslurm.slurm cimport ( + partition_info_msg_t, + job_defaults_t, + delete_part_msg_t, + partition_info_t, + update_part_msg_t, + slurm_free_partition_info_members, + slurm_free_partition_info_msg, + slurm_free_update_part_msg, + slurm_init_part_desc_msg, + slurm_load_partitions, + slurm_sprint_cpu_bind_type, + cpu_bind_type_t, + slurm_preempt_mode_string, + slurm_preempt_mode_num, + slurm_create_partition, + slurm_update_partition, + slurm_delete_partition, + xfree, + try_xmalloc, +) +from pyslurm.db.util cimport ( + SlurmList, + SlurmListItem, +) +from pyslurm.utils cimport cstr +from pyslurm.utils cimport ctime +from pyslurm.utils.ctime cimport time_t +from pyslurm.utils.uint cimport * +from pyslurm.core cimport slurmctld + + +cdef class Partitions(dict): + """A collection of [pyslurm.Partition][] objects. + + Args: + partitions (Union[list[str], dict[str, Partition], str], optional=None): + Partitions to initialize this collection with. + + Attributes: + total_cpus (int): + Total amount of CPUs the Partitions in a Collection have + total_nodes (int): + Total amount of Nodes the Partitions in a Collection have + """ + cdef: + partition_info_msg_t *info + partition_info_t tmp_info + + +cdef class Partition: + """A Slurm partition. + + ??? info "Setting Memory related attributes" + + Unless otherwise noted, all attributes in this class representing a + memory value, like `default_memory_per_cpu`, may also be set with a + string that contains suffixes like "K", "M", "G" or "T". + + For example: + + default_memory_per_cpu = "10G" + + This will internally be converted to 10240 (how the Slurm API expects + it) + + Args: + name (str, optional=None): + Name of a Partition + **kwargs (Any, optional=None): + Every attribute of a Partition can be set, except for: + + * total_cpus + * total_nodes + * select_type_parameters + * consumable_resource + + Attributes: + name (str): + Name of the Partition. + allowed_submit_nodes (list[str]): + List of Nodes from which Jobs can be submitted to the partition. + allowed_accounts (list[str]): + List of Accounts which are allowed to execute Jobs + allowed_groups (list[str]): + List of Groups which are allowed to execute Jobs + allowed_qos (list[str]): + List of QoS which are allowed to execute Jobs + alternate (str): + Name of the alternate Partition in case a Partition is down. + consumable_resource (str): + The type of consumable resource used in the Partition. + select_type_parameters (list[str]): + List of additional parameters passed to the select plugin used. + cpu_binding (str): + Default CPU-binding for Jobs that execute in a Partition. + default_memory_per_cpu (int): + Default Memory per CPU for Jobs in this Partition, in Mebibytes. + Mutually exclusive with `default_memory_per_node`. + + This can also return [UNLIMITED][pyslurm.constants.UNLIMITED] + default_memory_per_node (int): + Default Memory per Node for Jobs in this Partition, in Mebibytes. + Mutually exclusive with `default_memory_per_cpu`. + + This can also return [UNLIMITED][pyslurm.constants.UNLIMITED] + max_memory_per_cpu (int): + Max Memory per CPU allowed for Jobs in this Partition, in + Mebibytes. Mutually exclusive with `max_memory_per_node`. + + This can also return [UNLIMITED][pyslurm.constants.UNLIMITED] + max_memory_per_node (int): + Max Memory per Node allowed for Jobs in this Partition, in + Mebibytes. Mutually exclusive with `max_memory_per_cpu` + + This can also return [UNLIMITED][pyslurm.constants.UNLIMITED] + default_time (int): + Default run time-limit in minutes for Jobs that don't specify one. + + This can also return [UNLIMITED][pyslurm.constants.UNLIMITED] + denied_qos (list[str]): + List of QoS that cannot be used in a Partition + denied_accounts (list[str]): + List of Accounts that cannot use a Partition + preemption_grace_time (int): + Grace Time in seconds when a Job is selected for Preemption. + default_cpus_per_gpu (int): + Default CPUs per GPU for Jobs in this Partition + default_memory_per_gpu (int): + Default Memory per GPU, in Mebibytes, for Jobs in this Partition + max_cpus_per_node (int): + Max CPUs per Node allowed for Jobs in this Partition + + This can also return [UNLIMITED][pyslurm.constants.UNLIMITED] + max_cpus_per_socket (int): + Max CPUs per Socket allowed for Jobs in this Partition + + This can also return [UNLIMITED][pyslurm.constants.UNLIMITED] + max_nodes (int): + Max number of Nodes allowed for Jobs + + This can also return [UNLIMITED][pyslurm.constants.UNLIMITED] + min_nodes (int): + Minimum number of Nodes that must be requested by Jobs + max_time_limit (int): + Max Time-Limit in minutes that Jobs can request + + This can also return [UNLIMITED][pyslurm.constants.UNLIMITED] + oversubscribe (str): + The oversubscribe mode for this Partition + nodes (str): + Nodes that are in a Partition + nodesets (list[str]): + List of Nodesets that a Partition has configured + over_time_limit (int): + Limit in minutes that Jobs can exceed their time-limit + + This can also return [UNLIMITED][pyslurm.constants.UNLIMITED] + preempt_mode (str): + Preemption Mode in a Partition + priority_job_factor (int): + The Priority Job Factor for a partition + priority_tier (int): + The priority tier for a Partition + qos (str): + A QoS associated with a Partition, used to extend possible limits + total_cpus (int): + Total number of CPUs available in a Partition + total_nodes (int): + Total number of nodes available in a Partition + state (str): + State the Partition is in + is_default (bool): + Whether this Partition is the default partition or not + allow_root_jobs (bool): + Whether Jobs by the root user are allowed + is_user_exclusive (bool): + Whether nodes will be exclusively allocated to users + is_hidden (bool): + Whether the partition is hidden or not + least_loaded_nodes_scheduling (bool): + Whether Least-Loaded-Nodes scheduling algorithm is used on a + Partition + is_root_only (bool): + Whether only root is able to use a Partition + requires_reservation (bool): + Whether a reservation is required to use a Partition + """ + cdef: + partition_info_t *ptr + int power_save_enabled + slurmctld.Config slurm_conf + + @staticmethod + cdef Partition from_ptr(partition_info_t *in_ptr) diff --git a/pyslurm/core/partition.pyx b/pyslurm/core/partition.pyx new file mode 100644 index 00000000..25e17124 --- /dev/null +++ b/pyslurm/core/partition.pyx @@ -0,0 +1,866 @@ +######################################################################### +# partition.pyx - interface to work with partitions in slurm +######################################################################### +# Copyright (C) 2023 Toni Harzendorf +# Copyright (C) 2023 PySlurm Developers +# +# This file is part of PySlurm +# +# PySlurm is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# PySlurm is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with PySlurm; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# cython: c_string_type=unicode, c_string_encoding=default +# cython: language_level=3 + +from typing import Union, Any +from pyslurm.utils import cstr +from pyslurm.utils import ctime +from pyslurm.utils.uint import * +from pyslurm.core.error import RPCError, verify_rpc +from pyslurm.utils.ctime import timestamp_to_date, _raw_time +from pyslurm.constants import UNLIMITED +from pyslurm.utils.helpers import ( + uid_to_name, + gid_to_name, + _getgrall_to_dict, + _getpwall_to_dict, + cpubind_to_num, + instance_to_dict, + _sum_prop, + dehumanize, +) +from pyslurm.utils.ctime import ( + timestr_to_mins, + timestr_to_secs, +) + + +cdef class Partitions(dict): + def __dealloc__(self): + slurm_free_partition_info_msg(self.info) + + def __cinit__(self): + self.info = NULL + + def __init__(self, partitions=None): + if isinstance(partitions, dict): + self.update(partitions) + elif isinstance(partitions, str): + partlist = partitions.split(",") + self.update({part: Partition(part) for part in partlist}) + elif partitions is not None: + for part in partitions: + if isinstance(part, str): + self[part] = Partition(part) + else: + self[part.name] = part + + @staticmethod + def load(): + """Load all Partitions in the system. + + Returns: + (pyslurm.Partitions): Collection of Partition objects. + + Raises: + RPCError: When getting all the Partitions from the slurmctld + failed. + """ + cdef: + Partitions partitions = Partitions.__new__(Partitions) + int flags = slurm.SHOW_ALL + Partition partition + slurmctld.Config slurm_conf + int power_save_enabled = 0 + + verify_rpc(slurm_load_partitions(0, &partitions.info, flags)) + slurm_conf = slurmctld.Config.load() + + # zero-out a dummy partition_info_t + memset(&partitions.tmp_info, 0, sizeof(partition_info_t)) + + if slurm_conf.suspend_program and slurm_conf.resume_program: + power_save_enabled = 1 + + # Put each pointer into its own instance. + for cnt in range(partitions.info.record_count): + partition = Partition.from_ptr(&partitions.info.partition_array[cnt]) + + # Prevent double free if xmalloc fails mid-loop and a MemoryError + # is raised by replacing it with a zeroed-out partition_info_t. + partitions.info.partition_array[cnt] = partitions.tmp_info + + partition.power_save_enabled = power_save_enabled + partition.slurm_conf = slurm_conf + partitions[partition.name] = partition + + # At this point we memcpy'd all the memory for the Partitions. Setting + # this to 0 will prevent the slurm partition free function to + # deallocate the memory for the individual partitions. This should be + # fine, because they are free'd automatically in __dealloc__ since the + # lifetime of each partition-pointer is tied to the lifetime of its + # corresponding "Partition" instance. + partitions.info.record_count = 0 + + return partitions + + def reload(self): + """Reload the information for Partitions in a collection. + + !!! note + + Only information for Partitions which are already in the + collection at the time of calling this method will be reloaded. + + Returns: + (pyslurm.Partitions): Returns self + + Raises: + RPCError: When getting the Partitions from the slurmctld failed. + """ + cdef Partitions reloaded_parts + our_parts = list(self.keys()) + + if not our_parts: + return self + + reloaded_parts = Partitions.load() + for part in our_parts: + if part in reloaded_parts: + # Put the new data in. + self[part] = reloaded_parts[part] + + return self + + def set_state(self, state): + """Modify the State of all Partitions in this Collection. + + Args: + state (str): + Partition state to set + + Raises: + RPCError: When updating the state failed + """ + for part in self.values(): + part.modify(state=state) + + def as_list(self): + """Format the information as list of Partition objects. + + Returns: + (list): List of Partition objects + """ + return list(self.values()) + + @property + def total_cpus(self): + return _sum_prop(self, Partition.total_cpus) + + @property + def total_nodes(self): + return _sum_prop(self, Partition.total_nodes) + + +cdef class Partition: + + def __cinit__(self): + self.ptr = NULL + + def __init__(self, name=None, **kwargs): + self._alloc_impl() + self.name = name + for k, v in kwargs.items(): + setattr(self, k, v) + + def _alloc_impl(self): + if not self.ptr: + self.ptr = try_xmalloc(sizeof(partition_info_t)) + if not self.ptr: + raise MemoryError("xmalloc failed for partition_info_t") + + slurm_init_part_desc_msg(self.ptr) + + def _dealloc_impl(self): + slurm_free_partition_info_members(self.ptr) + xfree(self.ptr) + + def __dealloc__(self): + self._dealloc_impl() + + @staticmethod + cdef Partition from_ptr(partition_info_t *in_ptr): + cdef Partition wrap = Partition.__new__(Partition) + wrap._alloc_impl() + memcpy(wrap.ptr, in_ptr, sizeof(partition_info_t)) + return wrap + + def _error_or_name(self): + if not self.name: + raise ValueError("You need to set a Partition name for this " + "instance.") + return self.name + + def as_dict(self): + """Partition information formatted as a dictionary. + + Returns: + (dict): Partition information as dict + + Examples: + >>> import pyslurm + >>> mypart = pyslurm.Partition.load("mypart") + >>> mypart_dict = mypart.as_dict() + """ + return instance_to_dict(self) + + @staticmethod + def load(name): + """Load information for a specific Partition. + + Args: + name (str): + The name of the Partition to load. + + Returns: + (pyslurm.Partition): Returns a new Partition instance. + + Raises: + RPCError: If requesting the Partition information from the + slurmctld was not successful. + + Examples: + >>> import pyslurm + >>> part = pyslurm.Partition.load("normal") + """ + partitions = Partitions.load() + if name not in partitions: + raise RPCError(msg=f"Partition '{name}' doesn't exist") + + return partitions[name] + + def create(self): + """Create a Partition. + + Implements the slurm_create_partition RPC. + + Returns: + (pyslurm.Partition): This function returns the current Partition + instance object itself. + + Raises: + RPCError: If creating the Partition was not successful. + + Examples: + >>> import pyslurm + >>> part = pyslurm.Partition("debug").create() + """ + self._error_or_name() + verify_rpc(slurm_create_partition(self.ptr)) + return self + + def modify(self, **changes): + """Modify a Partition. + + Implements the slurm_update_partition RPC. + + Args: + **changes (Any): + Changes for the Partition. Almost every Attribute from a + Partition can be modified, except for: + + * total_cpus + * total_nodes + * select_type_parameters + * consumable_resource + + Raises: + ValueError: When no changes were specified or when a parsing error + occured. + RPCError: When updating the Partition was not successful. + + Examples: + >>> import pyslurm + >>> + >>> # Modifying the maximum time limit + >>> mypart = pyslurm.Partition("normal") + >>> mypart.modify(max_time_limit="10-00:00:00") + >>> + >>> # Modifying the partition state + >>> mypart.modify(state="DRAIN") + """ + if not changes: + raise ValueError("No changes were specified") + + cdef Partition part = Partition(**changes) + part.name = self._error_or_name() + verify_rpc(slurm_update_partition(part.ptr)) + + def delete(self): + """Delete a Partition. + + Implements the slurm_delete_partition RPC. + + Raises: + RPCError: When deleting the Partition was not successful. + + Examples: + >>> import pyslurm + >>> pyslurm.Partition("normal").delete() + """ + cdef delete_part_msg_t del_part_msg + memset(&del_part_msg, 0, sizeof(del_part_msg)) + + del_part_msg.name = cstr.from_unicode(self._error_or_name()) + verify_rpc(slurm_delete_partition(&del_part_msg)) + + # If using property getter/setter style internally becomes too messy at + # some point, we can easily switch to normal "cdef public" attributes and + # just extract the getter/setter logic into two functions, where one + # creates a pointer from the instance attributes, and the other parses + # pointer values into instance attributes. + # + # From a user perspective nothing would change. + + @property + def name(self): + return cstr.to_unicode(self.ptr.name) + + @name.setter + def name(self, val): + cstr.fmalloc(&self.ptr.name, val) + + @property + def allowed_submit_nodes(self): + return cstr.to_list(self.ptr.allow_alloc_nodes, ["ALL"]) + + @allowed_submit_nodes.setter + def allowed_submit_nodes(self, val): + cstr.from_list(&self.ptr.allow_alloc_nodes, val) + + @property + def allowed_accounts(self): + return cstr.to_list(self.ptr.allow_accounts, ["ALL"]) + + @allowed_accounts.setter + def allowed_accounts(self, val): + cstr.from_list(&self.ptr.allow_accounts, val) + + @property + def allowed_groups(self): + return cstr.to_list(self.ptr.allow_groups, ["ALL"]) + + @allowed_groups.setter + def allowed_groups(self, val): + cstr.from_list(&self.ptr.allow_groups, val) + + @property + def allowed_qos(self): + return cstr.to_list(self.ptr.allow_qos, ["ALL"]) + + @allowed_qos.setter + def allowed_qos(self, val): + cstr.from_list(&self.ptr.allow_qos, val) + + @property + def alternate(self): + return cstr.to_unicode(self.ptr.alternate) + + @alternate.setter + def alternate(self, val): + cstr.fmalloc(&self.ptr.alternate, val) + + @property + def consumable_resource(self): + return _select_type_int_to_cons_res(self.ptr.cr_type) + + @property + def select_type_parameters(self): + return _select_type_int_to_list(self.ptr.cr_type) + + @property + def cpu_binding(self): + cdef char cpu_bind[128] + slurm_sprint_cpu_bind_type(cpu_bind, + self.ptr.cpu_bind) + if cpu_bind == "(null type)": + return None + + return cstr.to_unicode(cpu_bind) + + @cpu_binding.setter + def cpu_binding(self, val): + self.ptr.cpu_bind = cpubind_to_num(val) + + @property + def default_memory_per_cpu(self): + return _get_memory(self.ptr.def_mem_per_cpu, per_cpu=True) + + @default_memory_per_cpu.setter + def default_memory_per_cpu(self, val): + self.ptr.def_mem_per_cpu = u64(dehumanize(val)) + self.ptr.def_mem_per_cpu |= slurm.MEM_PER_CPU + + @property + def default_memory_per_node(self): + return _get_memory(self.ptr.def_mem_per_cpu, per_cpu=False) + + @default_memory_per_node.setter + def default_memory_per_node(self, val): + self.ptr.def_mem_per_cpu = u64(dehumanize(val)) + + @property + def max_memory_per_cpu(self): + return _get_memory(self.ptr.max_mem_per_cpu, per_cpu=True) + + @max_memory_per_cpu.setter + def max_memory_per_cpu(self, val): + self.ptr.max_mem_per_cpu = u64(dehumanize(val)) + self.ptr.max_mem_per_cpu |= slurm.MEM_PER_CPU + + @property + def max_memory_per_node(self): + return _get_memory(self.ptr.max_mem_per_cpu, per_cpu=False) + + @max_memory_per_node.setter + def max_memory_per_node(self, val): + self.ptr.max_mem_per_cpu = u64(dehumanize(val)) + + @property + def default_time(self): + return _raw_time(self.ptr.default_time, on_inf=UNLIMITED) + + @default_time.setter + def default_time(self, val): + self.ptr.default_time = timestr_to_mins(val) + + @property + def denied_qos(self): + return cstr.to_list(self.ptr.deny_qos, ["ALL"]) + + @denied_qos.setter + def denied_qos(self, val): + cstr.from_list(&self.ptr.deny_qos, val) + + @property + def denied_accounts(self): + return cstr.to_list(self.ptr.deny_accounts, ["ALL"]) + + @denied_accounts.setter + def denied_accounts(self, val): + cstr.from_list(&self.ptr.deny_accounts, val) + + @property + def preemption_grace_time(self): + return _raw_time(self.ptr.grace_time) + + @preemption_grace_time.setter + def preemption_grace_time(self, val): + self.ptr.grace_time = timestr_to_secs(val) + + @property + def default_cpus_per_gpu(self): + def_dict = cstr.to_dict(self.ptr.job_defaults_str) + if def_dict and "DefCpuPerGpu" in def_dict: + return int(def_dict["DefCpuPerGpu"]) + + return _extract_job_default_item(slurm.JOB_DEF_CPU_PER_GPU, + self.ptr.job_defaults_list) + + @default_cpus_per_gpu.setter + def default_cpus_per_gpu(self, val): + _concat_job_default_str("DefCpuPerGpu", val, + &self.ptr.job_defaults_str) + + @property + def default_memory_per_gpu(self): + def_dict = cstr.to_dict(self.ptr.job_defaults_str) + if def_dict and "DefMemPerGpu" in def_dict: + return int(def_dict["DefMemPerGpu"]) + + return _extract_job_default_item(slurm.JOB_DEF_MEM_PER_GPU, + self.ptr.job_defaults_list) + + @default_memory_per_gpu.setter + def default_memory_per_gpu(self, val): + _concat_job_default_str("DefMemPerGpu", val, + &self.ptr.job_defaults_str) + + @property + def max_cpus_per_node(self): + return u32_parse(self.ptr.max_cpus_per_node) + + @max_cpus_per_node.setter + def max_cpus_per_node(self, val): + self.ptr.max_cpus_per_node = u32(val) + + @property + def max_cpus_per_socket(self): + return u32_parse(self.ptr.max_cpus_per_socket) + + @max_cpus_per_socket.setter + def max_cpus_per_socket(self, val): + self.ptr.max_cpus_per_socket = u32(val) + + @property + def max_nodes(self): + return u32_parse(self.ptr.max_nodes) + + @max_nodes.setter + def max_nodes(self, val): + self.ptr.max_nodes = u32(val) + + @property + def min_nodes(self): + return u32_parse(self.ptr.min_nodes, zero_is_noval=False) + + @min_nodes.setter + def min_nodes(self, val): + self.ptr.min_nodes = u32(val, zero_is_noval=False) + + @property + def max_time_limit(self): + return _raw_time(self.ptr.max_time, on_inf=UNLIMITED) + + @max_time_limit.setter + def max_time_limit(self, val): + self.ptr.max_time = timestr_to_mins(val) + + @property + def oversubscribe(self): + return _oversubscribe_int_to_str(self.ptr.max_share) + + @oversubscribe.setter + def oversubscribe(self, val): + self.ptr.max_share = _oversubscribe_str_to_int(val) + + @property + def nodes(self): + return cstr.to_unicode(self.ptr.nodes) + + @nodes.setter + def nodes(self, val): + cstr.from_list(&self.ptr.nodes, val) + + @property + def nodesets(self): + return cstr.to_list(self.ptr.nodesets) + + @nodesets.setter + def nodesets(self, val): + cstr.from_list(&self.ptr.nodesets, val) + + @property + def over_time_limit(self): + return u16_parse(self.ptr.over_time_limit) + + @over_time_limit.setter + def over_time_limit(self, val): + self.ptr.over_time_limit = u16(self.ptr.over_time_limit) + + @property + def preempt_mode(self): + return _preempt_mode_int_to_str(self.ptr.preempt_mode, self.slurm_conf) + + @preempt_mode.setter + def preempt_mode(self, val): + self.ptr.preempt_mode = _preempt_mode_str_to_int(val) + + @property + def priority_job_factor(self): + return u16_parse(self.ptr.priority_job_factor) + + @priority_job_factor.setter + def priority_job_factor(self, val): + self.ptr.priority_job_factor = u16(val) + + @property + def priority_tier(self): + return u16_parse(self.ptr.priority_tier) + + @priority_tier.setter + def priority_tier(self, val): + self.ptr.priority_tier = u16(val) + + @property + def qos(self): + return cstr.to_unicode(self.ptr.qos_char) + + @qos.setter + def qos(self, val): + cstr.fmalloc(&self.ptr.qos_char, val) + + @property + def total_cpus(self): + return u32_parse(self.ptr.total_cpus, on_noval=0) + + @property + def total_nodes(self): + return u32_parse(self.ptr.total_nodes, on_noval=0) + + @property + def state(self): + return _partition_state_int_to_str(self.ptr.state_up) + + @state.setter + def state(self, val): + self.ptr.state_up = _partition_state_str_to_int(val) + + @property + def is_default(self): + return u16_parse_bool_flag(self.ptr.flags, slurm.PART_FLAG_DEFAULT) + + @is_default.setter + def is_default(self, val): + u16_set_bool_flag(&self.ptr.flags, val, + slurm.PART_FLAG_DEFAULT, slurm.PART_FLAG_DEFAULT_CLR) + + @property + def allow_root_jobs(self): + return u16_parse_bool_flag(self.ptr.flags, slurm.PART_FLAG_NO_ROOT) + + @allow_root_jobs.setter + def allow_root_jobs(self, val): + u16_set_bool_flag(&self.ptr.flags, val, slurm.PART_FLAG_NO_ROOT, + slurm.PART_FLAG_NO_ROOT_CLR) + + @property + def is_user_exclusive(self): + return u16_parse_bool_flag(self.ptr.flags, + slurm.PART_FLAG_EXCLUSIVE_USER) + + @is_user_exclusive.setter + def is_user_exclusive(self, val): + u16_set_bool_flag(&self.ptr.flags, val, slurm.PART_FLAG_EXCLUSIVE_USER, + slurm.PART_FLAG_EXC_USER_CLR) + + @property + def is_hidden(self): + return u16_parse_bool_flag(self.ptr.flags, slurm.PART_FLAG_HIDDEN) + + @is_hidden.setter + def is_hidden(self, val): + u16_set_bool_flag(&self.ptr.flags, val, + slurm.PART_FLAG_HIDDEN, slurm.PART_FLAG_HIDDEN_CLR) + + @property + def least_loaded_nodes_scheduling(self): + return u16_parse_bool_flag(self.ptr.flags, slurm.PART_FLAG_LLN) + + @least_loaded_nodes_scheduling.setter + def least_loaded_nodes_scheduling(self, val): + u16_set_bool_flag(&self.ptr.flags, val, slurm.PART_FLAG_LLN, + slurm.PART_FLAG_LLN_CLR) + + @property + def is_root_only(self): + return u16_parse_bool_flag(self.ptr.flags, slurm.PART_FLAG_ROOT_ONLY) + + @is_root_only.setter + def is_root_only(self, val): + u16_set_bool_flag(&self.ptr.flags, val, slurm.PART_FLAG_ROOT_ONLY, + slurm.PART_FLAG_ROOT_ONLY_CLR) + + @property + def requires_reservation(self): + return u16_parse_bool_flag(self.ptr.flags, slurm.PART_FLAG_REQ_RESV) + + @requires_reservation.setter + def requires_reservation(self, val): + u16_set_bool_flag(&self.ptr.flags, val, slurm.PART_FLAG_REQ_RESV, + slurm.PART_FLAG_REQ_RESV_CLR) + + # TODO: tres_fmt_str + + +def _partition_state_int_to_str(state): + if state == slurm.PARTITION_UP: + return "UP" + elif state == slurm.PARTITION_DOWN: + return "DOWN" + elif state == slurm.PARTITION_INACTIVE: + return "INACTIVE" + elif state == slurm.PARTITION_DRAIN: + return "DRAIN" + else: + return "UNKNOWN" + + +def _partition_state_str_to_int(state): + state = state.upper() + + if state == "UP": + return slurm.PARTITION_UP + elif state == "DOWN": + return slurm.PARTITION_DOWN + elif state == "INACTIVE": + return slurm.PARTITION_INACTIVE + elif state == "DRAIN": + return slurm.PARTITION_DRAIN + else: + choices = "UP, DOWN, INACTIVE, DRAIN" + raise ValueError(f"Invalid partition state: {state}, valid choices " + f"are {choices}") + + +def _oversubscribe_int_to_str(shared): + if shared == slurm.NO_VAL16: + return None + + is_forced = shared & slurm.SHARED_FORCE + max_jobs = shared & (~slurm.SHARED_FORCE) + + if not max_jobs: + return "EXCLUSIVE" + elif is_forced: + return f"FORCE:{max_jobs}" + elif max_jobs == 1: + return "NO" + else: + return f"YES:{max_jobs}" + + +def _oversubscribe_str_to_int(typ): + typ = typ.upper() + + if typ == "NO": + return 1 + elif typ == "EXCLUSIVE": + return 0 + elif "YES" in typ: + return _split_oversubscribe_str(typ) + elif "FORCE" in typ: + return _split_oversubscribe_str(typ) | slurm.SHARED_FORCE + else: + return slurm.NO_VAL16 + + +def _split_oversubscribe_str(val): + max_jobs = val.split(":", 1) + if len(max_jobs) == 2: + return int(max_jobs[1]) + else: + return 4 + + +def _select_type_int_to_list(stype): + # The rest of the CR_* stuff are just some extra parameters to the select + # plugin + out = [] + + if stype & slurm.CR_OTHER_CONS_RES: + out.append("OTHER_CONS_RES") + + if stype & slurm.CR_ONE_TASK_PER_CORE: + out.append("ONE_TASK_PER_CORE") + + if stype & slurm.CR_PACK_NODES: + out.append("PACK_NODES") + + if stype & slurm.CR_OTHER_CONS_TRES: + out.append("OTHER_CONS_TRES") + + if stype & slurm.CR_CORE_DEFAULT_DIST_BLOCK: + out.append("CORE_DEFAULT_DIST_BLOCK") + + if stype & slurm.CR_LLN: + out.append("LLN") + + return out + + +def _select_type_int_to_cons_res(stype): + # https://github.com/SchedMD/slurm/blob/257ca5e4756a493dc4c793ded3ac3c1a769b3c83/slurm/slurm.h#L996 + # The 3 main select types are mutually exclusive, and may be combined with + # CR_MEMORY + # CR_BOARD exists but doesn't show up in the documentation, so ignore it. + if stype & slurm.CR_CPU and stype & slurm.CR_MEMORY: + return "CPU_MEMORY" + elif stype & slurm.CR_CORE and stype & slurm.CR_MEMORY: + return "CORE_MEMORY" + elif stype & slurm.CR_SOCKET and stype & slurm.CR_MEMORY: + return "SOCKET_MEMORY" + elif stype & slurm.CR_CPU: + return "CPU" + elif stype & slurm.CR_CORE: + return "CORE" + elif stype & slurm.CR_SOCKET: + return "SOCKET" + elif stype & slurm.CR_MEMORY: + return "MEMORY" + else: + return None + + +def _preempt_mode_str_to_int(mode): + if not mode: + return slurm.NO_VAL16 + + pmode = slurm_preempt_mode_num(str(mode)) + if pmode == slurm.NO_VAL16: + raise ValueError(f"Invalid Preempt mode: {mode}") + + return pmode + + +def _preempt_mode_int_to_str(mode, slurmctld.Config slurm_conf): + if mode == slurm.NO_VAL16: + return slurm_conf.preempt_mode if slurm_conf else None + else: + return cstr.to_unicode(slurm_preempt_mode_string(mode)) + + +cdef _extract_job_default_item(typ, slurm.List job_defaults_list): + cdef: + job_defaults_t *default_item + SlurmList job_def_list + SlurmListItem job_def_item + + job_def_list = SlurmList.wrap(job_defaults_list, owned=False) + for job_def_item in job_def_list: + default_item = job_def_item.data + if default_item.type == typ: + return default_item.value + + return None + + +cdef _concat_job_default_str(typ, val, char **job_defaults_str): + cdef uint64_t _val = u64(dehumanize(val)) + + current = cstr.to_dict(job_defaults_str[0]) + if _val == slurm.NO_VAL64: + current.pop(typ, None) + else: + current.update({typ : _val}) + + cstr.from_dict(job_defaults_str, current) + + +def _get_memory(value, per_cpu): + if value != slurm.NO_VAL64: + if value & slurm.MEM_PER_CPU and per_cpu: + if value == slurm.MEM_PER_CPU: + return UNLIMITED + return u64_parse(value & (~slurm.MEM_PER_CPU)) + + # For these values, Slurm interprets 0 as being equal to + # INFINITE/UNLIMITED + elif value == 0 and not per_cpu: + return UNLIMITED + + elif not value & slurm.MEM_PER_CPU and not per_cpu: + return u64_parse(value) + + return None diff --git a/pyslurm/core/slurmctld.pxd b/pyslurm/core/slurmctld.pxd index 0f42fffb..8bafb01f 100644 --- a/pyslurm/core/slurmctld.pxd +++ b/pyslurm/core/slurmctld.pxd @@ -27,6 +27,7 @@ from pyslurm.slurm cimport ( slurm_conf_t, slurm_load_ctl_conf, slurm_free_ctl_conf, + slurm_preempt_mode_string, try_xmalloc, ) from pyslurm.utils cimport cstr diff --git a/pyslurm/core/slurmctld.pyx b/pyslurm/core/slurmctld.pyx index 2b5367c5..7f06966e 100644 --- a/pyslurm/core/slurmctld.pyx +++ b/pyslurm/core/slurmctld.pyx @@ -46,3 +46,17 @@ cdef class Config: @property def cluster(self): return cstr.to_unicode(self.ptr.cluster_name) + + @property + def preempt_mode(self): + cdef char *tmp = slurm_preempt_mode_string(self.ptr.preempt_mode) + return cstr.to_unicode(tmp) + + @property + def suspend_program(self): + return cstr.to_unicode(self.ptr.suspend_program) + + @property + def resume_program(self): + return cstr.to_unicode(self.ptr.resume_program) + diff --git a/pyslurm/slurm/extra.pxi b/pyslurm/slurm/extra.pxi index 0ccb0708..c18db9dc 100644 --- a/pyslurm/slurm/extra.pxi +++ b/pyslurm/slurm/extra.pxi @@ -271,3 +271,10 @@ cdef extern char *slurm_hostlist_deranged_string_malloc(hostlist_t hl) cdef extern void slurmdb_job_cond_def_start_end(slurmdb_job_cond_t *job_cond) cdef extern uint64_t slurmdb_find_tres_count_in_string(char *tres_str_in, int id) + +# +# Slurm Partition functions +# + +cdef extern void slurm_free_update_part_msg(update_part_msg_t *msg) +cdef extern void slurm_free_partition_info_members(partition_info_t *node) diff --git a/pyslurm/utils/cstr.pxd b/pyslurm/utils/cstr.pxd index b1719bde..e8014a5f 100644 --- a/pyslurm/utils/cstr.pxd +++ b/pyslurm/utils/cstr.pxd @@ -31,7 +31,7 @@ cdef to_unicode(char *s, default=*) cdef fmalloc(char **old, val) cdef fmalloc2(char **p1, char **p2, val) cdef free_array(char **arr, count) -cpdef list to_list(char *str_list) +cpdef list to_list(char *str_list, default=*) cdef from_list(char **old, vals, delim=*) cdef from_list2(char **p1, char **p2, vals, delim=*) cpdef dict to_dict(char *str_dict, str delim1=*, str delim2=*) diff --git a/pyslurm/utils/cstr.pyx b/pyslurm/utils/cstr.pyx index 12a39ecb..489d80e8 100644 --- a/pyslurm/utils/cstr.pyx +++ b/pyslurm/utils/cstr.pyx @@ -46,7 +46,7 @@ cdef to_unicode(char *_str, default=None): """Convert a char* to Python3 str (unicode)""" if _str and _str[0] != NULL_BYTE: if _str == NONE_BYTE: - return None + return default return _str else: @@ -96,12 +96,12 @@ cdef fmalloc(char **old, val): old[0] = NULL -cpdef list to_list(char *str_list): +cpdef list to_list(char *str_list, default=[]): """Convert C-String to a list.""" cdef str ret = to_unicode(str_list) if not ret: - return [] + return default return ret.split(",") @@ -137,7 +137,7 @@ cpdef dict to_dict(char *str_dict, str delim1=",", str delim2="="): str key, val dict out = {} - if not _str_dict or delim1 not in _str_dict: + if not _str_dict: return out for kv in _str_dict.split(delim1): @@ -187,7 +187,7 @@ def dict_to_str(vals, prepend=None, delim1=",", delim2="="): for k, v in tmp_dict.items(): if ((delim1 in k or delim2 in k) or - delim1 in v or delim2 in v): + delim1 in str(v) or delim2 in str(v)): raise ValueError( f"Key or Value cannot contain either {delim1} or {delim2}. " f"Got Key: {k} and Value: {v}." diff --git a/pyslurm/utils/ctime.pyx b/pyslurm/utils/ctime.pyx index 5ffbc424..45d7c8e2 100644 --- a/pyslurm/utils/ctime.pyx +++ b/pyslurm/utils/ctime.pyx @@ -23,6 +23,7 @@ # cython: language_level=3 import datetime +from pyslurm.constants import UNLIMITED def timestr_to_secs(timestr): @@ -41,7 +42,7 @@ def timestr_to_secs(timestr): if timestr is None: return slurm.NO_VAL - elif timestr == "unlimited": + elif timestr == UNLIMITED or timestr.casefold() == "unlimited": return slurm.INFINITE if str(timestr).isdigit(): @@ -72,7 +73,9 @@ def timestr_to_mins(timestr): if timestr is None: return slurm.NO_VAL - elif timestr == "unlimited": + elif str(timestr).isdigit(): + return timestr + elif timestr == UNLIMITED or timestr.casefold() == "unlimited": return slurm.INFINITE tmp = cstr.from_unicode(timestr) @@ -111,7 +114,7 @@ def secs_to_timestr(secs, default=None): else: return tmp else: - return "unlimited" + return UNLIMITED def mins_to_timestr(mins, default=None): @@ -141,7 +144,7 @@ def mins_to_timestr(mins, default=None): else: return tmp else: - return "unlimited" + return UNLIMITED def date_to_timestamp(date, on_nodate=0): @@ -204,10 +207,10 @@ def timestamp_to_date(timestamp): return ret -def _raw_time(time, default=None): - if (time == slurm.NO_VAL or - time == 0 or - time == slurm.INFINITE): - return default - - return time +def _raw_time(time, on_noval=None, on_inf=None): + if time == slurm.NO_VAL or time == 0: + return on_noval + elif time == slurm.INFINITE: + return on_inf + else: + return time diff --git a/pyslurm/utils/helpers.pyx b/pyslurm/utils/helpers.pyx index 3617112e..28604422 100644 --- a/pyslurm/utils/helpers.pyx +++ b/pyslurm/utils/helpers.pyx @@ -28,6 +28,7 @@ from os import getuid, getgid from itertools import chain import re import signal +from pyslurm.constants import UNLIMITED MEMORY_UNITS = { @@ -235,7 +236,7 @@ def humanize(num, decimals=1): Returns: (str): Humanized number with appropriate suffix. """ - if num is None or num == "unlimited": + if num is None or num == "unlimited" or num == UNLIMITED: return num num = int(num) diff --git a/pyslurm/utils/uint.pxd b/pyslurm/utils/uint.pxd index 0fd38739..3d8f50e5 100644 --- a/pyslurm/utils/uint.pxd +++ b/pyslurm/utils/uint.pxd @@ -35,9 +35,13 @@ cpdef u32_parse(uint32_t val, on_inf=*, on_noval=*, noval=*, zero_is_noval=*) cpdef u64_parse(uint64_t val, on_inf=*, on_noval=*, noval=*, zero_is_noval=*) cpdef u8_bool(val) cpdef u16_bool(val) +cdef uint_set_bool_flag(flags, boolean, true_flag, false_flag=*) +cdef uint_parse_bool_flag(flags, flag, no_val) +cdef uint_parse_bool(val, no_val) +cdef uint_bool(val, no_val) cdef u8_parse_bool(uint8_t val) cdef u16_parse_bool(uint16_t val) cdef u64_parse_bool_flag(uint64_t flags, flag) -cdef u64_set_bool_flag(uint64_t *flags, boolean, flag_val) +cdef u64_set_bool_flag(uint64_t *flags, boolean, true_flag, false_flag=*) cdef u16_parse_bool_flag(uint16_t flags, flag) -cdef u16_set_bool_flag(uint16_t *flags, boolean, flag_val) +cdef u16_set_bool_flag(uint16_t *flags, boolean, true_flag, false_flag=*) diff --git a/pyslurm/utils/uint.pyx b/pyslurm/utils/uint.pyx index 7418e109..0dae7779 100644 --- a/pyslurm/utils/uint.pyx +++ b/pyslurm/utils/uint.pyx @@ -22,12 +22,14 @@ # cython: c_string_type=unicode, c_string_encoding=default # cython: language_level=3 +from pyslurm.constants import UNLIMITED + cpdef u8(val, inf=False, noval=slurm.NO_VAL8, on_noval=slurm.NO_VAL8, zero_is_noval=True): """Try to convert arbitrary 'val' to uint8_t""" if val is None or (val == 0 and zero_is_noval) or val == noval: return on_noval - elif inf and val == "unlimited": + elif inf and (val == UNLIMITED or val == "unlimited"): return slurm.INFINITE8 else: if isinstance(val, str) and val.isdigit(): @@ -36,7 +38,7 @@ cpdef u8(val, inf=False, noval=slurm.NO_VAL8, on_noval=slurm.NO_VAL8, zero_is_no return val -cpdef u8_parse(uint8_t val, on_inf="unlimited", on_noval=None, noval=slurm.NO_VAL8, zero_is_noval=True): +cpdef u8_parse(uint8_t val, on_inf=UNLIMITED, on_noval=None, noval=slurm.NO_VAL8, zero_is_noval=True): """Convert uint8_t to Python int (with a few situational parameters)""" if val == noval or (val == 0 and zero_is_noval): return on_noval @@ -50,7 +52,7 @@ cpdef u16(val, inf=False, noval=slurm.NO_VAL16, on_noval=slurm.NO_VAL16, zero_is """Try to convert arbitrary 'val' to uint16_t""" if val is None or (val == 0 and zero_is_noval) or val == noval: return on_noval - elif inf and val == "unlimited": + elif inf and (val == UNLIMITED or val == "unlimited"): return slurm.INFINITE16 else: if isinstance(val, str) and val.isdigit(): @@ -59,7 +61,7 @@ cpdef u16(val, inf=False, noval=slurm.NO_VAL16, on_noval=slurm.NO_VAL16, zero_is return val -cpdef u16_parse(uint16_t val, on_inf="unlimited", on_noval=None, noval=slurm.NO_VAL16, zero_is_noval=True): +cpdef u16_parse(uint16_t val, on_inf=UNLIMITED, on_noval=None, noval=slurm.NO_VAL16, zero_is_noval=True): """Convert uint16_t to Python int (with a few situational parameters)""" if val == noval or (val == 0 and zero_is_noval): return on_noval @@ -73,7 +75,7 @@ cpdef u32(val, inf=False, noval=slurm.NO_VAL, on_noval=slurm.NO_VAL, zero_is_nov """Try to convert arbitrary 'val' to uint32_t""" if val is None or (val == 0 and zero_is_noval) or val == noval: return on_noval - elif inf and val == "unlimited": + elif inf and (val == UNLIMITED or val == "unlimited"): return slurm.INFINITE else: if isinstance(val, str) and val.isdigit(): @@ -82,7 +84,7 @@ cpdef u32(val, inf=False, noval=slurm.NO_VAL, on_noval=slurm.NO_VAL, zero_is_nov return val -cpdef u32_parse(uint32_t val, on_inf="unlimited", on_noval=None, noval=slurm.NO_VAL, zero_is_noval=True): +cpdef u32_parse(uint32_t val, on_inf=UNLIMITED, on_noval=None, noval=slurm.NO_VAL, zero_is_noval=True): """Convert uint32_t to Python int (with a few situational parameters)""" if val == noval or (val == 0 and zero_is_noval): return on_noval @@ -96,7 +98,7 @@ cpdef u64(val, inf=False, noval=slurm.NO_VAL64, on_noval=slurm.NO_VAL64, zero_is """Try to convert arbitrary 'val' to uint64_t""" if val is None or (val == 0 and zero_is_noval) or val == noval: return on_noval - elif inf and val == "unlimited": + elif inf and (val == UNLIMITED or val == "unlimited"): return slurm.INFINITE64 else: if isinstance(val, str) and val.isdigit(): @@ -105,7 +107,7 @@ cpdef u64(val, inf=False, noval=slurm.NO_VAL64, on_noval=slurm.NO_VAL64, zero_is return val -cpdef u64_parse(uint64_t val, on_inf="unlimited", on_noval=None, noval=slurm.NO_VAL64, zero_is_noval=True): +cpdef u64_parse(uint64_t val, on_inf=UNLIMITED, on_noval=None, noval=slurm.NO_VAL64, zero_is_noval=True): """Convert uint64_t to Python int (with a few situational parameters)""" if val == noval or (val == 0 and zero_is_noval): return on_noval @@ -115,67 +117,72 @@ cpdef u64_parse(uint64_t val, on_inf="unlimited", on_noval=None, noval=slurm.NO_ return val -cpdef u8_bool(val): - if val is None: - return slurm.NO_VAL8 - elif val: - return 1 +cdef uint_set_bool_flag(flags, boolean, true_flag, false_flag=0): + if boolean: + if false_flag: + flags &= ~false_flag + flags |= true_flag + elif boolean is not None: + if false_flag: + flags |= false_flag + flags &= ~true_flag + + return flags + + +cdef uint_parse_bool_flag(flags, flag, no_val): + if flags == no_val: + return False + + if flags & flag: + return True else: - return 0 + return False -cpdef u16_bool(val): +cdef uint_parse_bool(val, no_val): + if not val or val == no_val: + return False + + return True + + +cdef uint_bool(val, no_val): if val is None: - return slurm.NO_VAL16 + return no_val elif val: return 1 else: return 0 -cdef u8_parse_bool(uint8_t val): - if not val or val == slurm.NO_VAL8: - return False - - return True +cpdef u8_bool(val): + return uint_bool(val, slurm.NO_VAL8) -cdef u16_parse_bool(uint16_t val): - if not val or val == slurm.NO_VAL16: - return False +cpdef u16_bool(val): + return uint_bool(val, slurm.NO_VAL16) - return True +cdef u8_parse_bool(uint8_t val): + return uint_parse_bool(val, slurm.NO_VAL8) -cdef u64_set_bool_flag(uint64_t *flags, boolean, flag_val): - if boolean: - flags[0] |= flag_val - else: - flags[0] &= ~flag_val +cdef u16_parse_bool(uint16_t val): + return uint_parse_bool(val, slurm.NO_VAL16) -cdef u64_parse_bool_flag(uint64_t flags, flag): - if flags == slurm.NO_VAL: - return False - if flags & flag: - return True - else: - return False +cdef u16_set_bool_flag(uint16_t *flags, boolean, true_flag, false_flag=0): + flags[0] = uint_set_bool_flag(flags[0], boolean, true_flag, false_flag) -cdef u16_set_bool_flag(uint16_t *flags, boolean, flag_val): - if boolean: - flags[0] |= flag_val - else: - flags[0] &= ~flag_val +cdef u64_set_bool_flag(uint64_t *flags, boolean, true_flag, false_flag=0): + flags[0] = uint_set_bool_flag(flags[0], boolean, true_flag, false_flag) cdef u16_parse_bool_flag(uint16_t flags, flag): - if flags == slurm.NO_VAL16: - return False + return uint_parse_bool_flag(flags, flag, slurm.NO_VAL16) - if flags & flag: - return True - else: - return False + +cdef u64_parse_bool_flag(uint64_t flags, flag): + return uint_parse_bool_flag(flags, flag, slurm.NO_VAL64) diff --git a/tests/integration/test_partition.py b/tests/integration/test_partition.py new file mode 100644 index 00000000..bc5a28e2 --- /dev/null +++ b/tests/integration/test_partition.py @@ -0,0 +1,89 @@ +######################################################################### +# test_partition.py - partition api integration tests +######################################################################### +# Copyright (C) 2023 Toni Harzendorf +# Copyright (C) 2023 PySlurm Developers +# +# This file is part of PySlurm +# +# PySlurm is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# PySlurm is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with PySlurm; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +"""test_partition.py - Test the Partition api functions.""" + +import pytest +import pyslurm +import util +from pyslurm import Partition, Partitions, RPCError + + +def test_load(): + part = Partitions.load().as_list()[0] + + assert part.name + assert part.state + + with pytest.raises(RPCError, + match=f"Partition 'nonexistent' doesn't exist"): + Partition.load("nonexistent") + + +def test_create_delete(): + part = Partition( + name="testpart", + default_time="20-00:00:00", + default_memory_per_cpu=1024, + ) + part.create() + part.delete() + + +def test_modify(): + part = Partitions.load().as_list()[0] + + part.modify(default_time=120) + assert Partition.load(part.name).default_time == 120 + + part.modify(default_time="1-00:00:00") + assert Partition.load(part.name).default_time == 24*60 + + part.modify(default_time="UNLIMITED") + assert Partition.load(part.name).default_time == "UNLIMITED" + + part.modify(state="DRAIN") + assert Partition.load(part.name).state == "DRAIN" + + part.modify(state="UP") + assert Partition.load(part.name).state == "UP" + + +def test_parse_all(): + Partitions.load().as_list()[0].as_dict() + + +def test_reload(): + _partnames = [util.randstr() for i in range(3)] + _tmp_parts = Partitions(_partnames) + for part in _tmp_parts.values(): + part.create() + + all_parts = Partitions.load() + assert len(all_parts) >= 3 + + my_parts = Partitions(_partnames[1:]).reload() + assert len(my_parts) == 2 + for part in my_parts.as_list(): + assert part.state != "UNKNOWN" + + for part in _tmp_parts.values(): + part.delete() diff --git a/tests/integration/util.py b/tests/integration/util.py index f5032f1a..05576052 100644 --- a/tests/integration/util.py +++ b/tests/integration/util.py @@ -25,6 +25,7 @@ JobSubmitDescription, ) import time +import random, string # Horrendous, but works for now, because when testing against a real slurmctld # we need to wait a bit for state changes (i.e. we cancel a job and @@ -37,6 +38,11 @@ def wait(secs=WAIT_SECS_SLURMCTLD): time.sleep(secs) +def randstr(strlen=10): + chars = string.ascii_lowercase + return ''.join(random.choice(chars) for n in range(strlen)) + + def create_job_script(): job_script = """\ #!/bin/bash diff --git a/tests/unit/test_common.py b/tests/unit/test_common.py index dd812665..47832436 100644 --- a/tests/unit/test_common.py +++ b/tests/unit/test_common.py @@ -23,7 +23,7 @@ import pyslurm import pytest from datetime import datetime -from pyslurm import Job, JobSubmitDescription, Node +from pyslurm import Job, JobSubmitDescription, Node, Partition from pyslurm.utils.ctime import ( timestr_to_mins, timestr_to_secs, @@ -162,8 +162,8 @@ def _uint_impl(self, func_set, func_get, typ): val = func_set(str(2**typ-2)) assert func_get(val) == None - val = func_set("unlimited", inf=True) - assert func_get(val) == "unlimited" + val = func_set("UNLIMITED", inf=True) + assert func_get(val) == "UNLIMITED" val = func_set(0) assert func_get(val) == None @@ -173,7 +173,7 @@ def _uint_impl(self, func_set, func_get, typ): with pytest.raises(TypeError, match="an integer is required"): - val = func_set("unlimited") + val = func_set("UNLIMITED") with pytest.raises(OverflowError, match=r"can't convert negative value to*"): @@ -196,6 +196,28 @@ def test_u32(self): def test_u64(self): self._uint_impl(u64, u64_parse, 64) + def test_set_parse_bool_flag(self): + part = pyslurm.Partition() + + assert not part.is_hidden + + part.is_hidden = True + assert part.is_hidden + + part.is_root_only = True + assert part.is_hidden + assert part.is_root_only + assert not part.is_default + assert not part.allow_root_jobs + + part.is_default = False + part.is_hidden = False + assert not part.is_hidden + assert part.is_root_only + assert not part.is_default + assert not part.allow_root_jobs + + # def _uint_bool_impl(self, arg): # js = JobSubmitDescription() @@ -229,11 +251,11 @@ def test_parse_minutes(self): mins_str = "01:00:00" assert timestr_to_mins(mins_str) == mins - assert timestr_to_mins("unlimited") == 2**32-1 + assert timestr_to_mins("UNLIMITED") == 2**32-1 assert timestr_to_mins(None) == 2**32-2 assert mins_to_timestr(mins) == mins_str - assert mins_to_timestr(2**32-1) == "unlimited" + assert mins_to_timestr(2**32-1) == "UNLIMITED" assert mins_to_timestr(2**32-2) == None assert mins_to_timestr(0) == None @@ -246,11 +268,11 @@ def test_parse_seconds(self): secs_str = "01:00:00" assert timestr_to_secs(secs_str) == secs - assert timestr_to_secs("unlimited") == 2**32-1 + assert timestr_to_secs("UNLIMITED") == 2**32-1 assert timestr_to_secs(None) == 2**32-2 assert secs_to_timestr(secs) == secs_str - assert secs_to_timestr(2**32-1) == "unlimited" + assert secs_to_timestr(2**32-1) == "UNLIMITED" assert secs_to_timestr(2**32-2) == None assert secs_to_timestr(0) == None @@ -327,8 +349,8 @@ def test_humanize(self): val = humanize(800) assert val == "800.0M" - val = humanize("unlimited") - assert val == "unlimited" + val = humanize("UNLIMITED") + assert val == "UNLIMITED" val = humanize(None) assert val == None diff --git a/tests/unit/test_partition.py b/tests/unit/test_partition.py new file mode 100644 index 00000000..141a6e51 --- /dev/null +++ b/tests/unit/test_partition.py @@ -0,0 +1,98 @@ +######################################################################### +# test_partition.py - partition unit tests +######################################################################### +# Copyright (C) 2023 Toni Harzendorf +# Copyright (C) 2023 PySlurm Developers +# +# This file is part of PySlurm +# +# PySlurm is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# PySlurm is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with PySlurm; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +"""test_partition.py - Unit Test basic functionality of the Partition class.""" + +import pytest +import pyslurm +from pyslurm import Partition, Partitions + + +def test_create_instance(): + part = Partition("normal") + assert part.name == "normal" + + +def test_create_collection(): + parts = Partitions("part1,part2") + assert len(parts) == 2 + assert "part1" in parts + assert "part2" in parts + assert parts["part1"].name == "part1" + assert parts["part2"].name == "part2" + + parts = Partitions(["part1", "part2"]) + assert len(parts) == 2 + assert "part1" in parts + assert "part2" in parts + assert parts["part1"].name == "part1" + assert parts["part2"].name == "part2" + + parts = Partitions( + { + "part1": Partition("part1"), + "part2": Partition("part2"), + } + ) + assert len(parts) == 2 + assert "part1" in parts + assert "part2" in parts + assert parts["part1"].name == "part1" + assert parts["part2"].name == "part2" + + +def test_parse_all(): + Partition("normal").as_dict() + + +def test_parse_memory(): + part = Partition() + + assert part.default_memory_per_cpu is None + assert part.default_memory_per_node is None + + part.default_memory_per_cpu = "2G" + assert part.default_memory_per_cpu == 2048 + assert part.default_memory_per_node is None + + part.default_memory_per_node = "2G" + assert part.default_memory_per_cpu is None + assert part.default_memory_per_node == 2048 + + +def test_parse_job_defaults(): + part = Partition() + + assert part.default_cpus_per_gpu is None + assert part.default_memory_per_gpu is None + + part.default_cpus_per_gpu = 10 + assert part.default_cpus_per_gpu == 10 + assert part.default_memory_per_gpu is None + + part.default_memory_per_gpu = "10G" + assert part.default_cpus_per_gpu == 10 + assert part.default_memory_per_gpu == 10240 + + part.default_cpus_per_gpu = None + part.default_memory_per_gpu = None + assert part.default_cpus_per_gpu is None + assert part.default_memory_per_gpu is None