diff --git a/README.md b/README.md index 9e92dc43..1db47f45 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,23 @@ -# PySlurm +# PySlurm Logo -[![PySlurm](https://github.com/PySlurm/pyslurm/actions/workflows/pyslurm.yml/badge.svg?branch=main)](https://github.com/PySlurm/pyslurm/actions/workflows/pyslurm.yml) +pyslurm is the Python client library for the [Slurm Workload Manager](https://slurm.schedmd.com) -## Overview - -PySlurm is the Python client library for the [Slurm](https://slurm.schedmd.com) HPC Scheduler. - -## Prerequisites +## Requirements * [Slurm](https://slurm.schedmd.com) - Slurm shared library and header files * [Python](https://www.python.org) - >= 3.6 * [Cython](https://cython.org) - >= 0.29.30 but < 3.0 -This PySlurm branch is for the Slurm Major-Release 23.02 +This Version is for Slurm 23.02.x + +## Versioning + +In pyslurm, the versioning scheme follows the official Slurm versioning. The +first two numbers (`MAJOR.MINOR`) always correspond to Slurms Major-Release, +for example `23.02`. +The last number (`MICRO`) is however not tied in any way to Slurms `MICRO` +version, but is instead PySlurm's internal Patch-Level. For example, any +pyslurm 23.02.X version should work with any Slurm 23.02.X release. ## Installation @@ -21,14 +26,14 @@ By default, it is searched inside `/usr/include` for the Header files and in For Slurm installations in different locations, you will need to provide the corresponding paths to the necessary files. -You can specify these Paths with environment variables (recommended), for example: +You can specify those with environment variables (recommended), for example: ```shell export SLURM_INCLUDE_DIR=/opt/slurm/23.02/include export SLURM_LIB_DIR=/opt/slurm/23.02/lib ``` -Then you can proceed to install PySlurm, for example by cloning the Repository: +Then you can proceed to install pyslurm, for example by cloning the Repository: ```shell git clone https://github.com/PySlurm/pyslurm.git && cd pyslurm @@ -40,105 +45,14 @@ pip install . Also see `python setup.py --help` -## Release Versioning - -PySlurm's versioning scheme follows the official Slurm versioning. The first -two numbers (MAJOR.MINOR) always correspond to Slurms Major-Release, for example -`23.02`. The last number (MICRO) is however not tied in any way to Slurms -MICRO version. For example, any PySlurm 23.02.X version should work with any -Slurm 23.02.X release. - -## Documentation - -The API documentation is hosted at . - -To build the docs locally, use [Sphinx](http://www.sphinx-doc.org) to generate -the documentation from the reStructuredText based docstrings found in the -pyslurm module once it is built: - -```shell -cd doc -make clean -make html -``` - -## Testing - -PySlurm requires an installation of Slurm. - -### Using a Test Container - -To run tests locally without an existing Slurm cluster, `docker` and -`docker-compose` is required. - -Clone the project: - -```shell -git clone https://github.com/PySlurm/pyslurm.git -cd pyslurm -``` - -Start the Slurm container in the background: - -```shell -docker-compose up -d -``` - -The cluster takes a few seconds to start all the required Slurm services. Tail -the logs: - -```shell -docker-compose logs -f -``` - -When the cluster is ready, you will see the following log message: - -```text -Cluster is now available -``` - -Press CTRL+C to stop tailing the logs. Slurm is now running in a container in -detached mode. `docker-compose` also bind mounds the git directory inside the -container at `/pyslurm` so that the container has access to the test cases. - -Install test dependencies: - -```shell -pipenv sync --dev -``` - -Execute the tests inside the container: - -```shell -pipenv run pytest -sv scripts/run_tests_in_container.py -``` - -When testing is complete, stop the running Slurm container: - -```shell -docker-compose down -``` - -### Testing on an Existing Slurm Cluster - -You may also choose to clone the project and run tests on a node where Slurm is -already compiled and installed: - -```shell -git clone https://github.com/PySlurm/pyslurm.git -cd pyslurm -pip install . -./scripts/configure.sh -pipenv sync --dev -pipenv run pytest -sv -``` - ## Contributors -PySlurm is made by [contributors like +pyslurm is made by [contributors like you](https://github.com/PySlurm/pyslurm/graphs/contributors). -## Help +## Support + +Feel free to ask questions in the [GitHub +Discussions](https://github.com/orgs/PySlurm/discussions) -Ask questions on the [PySlurm Google -Group](https://groups.google.com/forum/#!forum/pyslurm) +Found a bug or you are missing a feature? Feel free to [open an Issue!](https://github.com/PySlurm/pyslurm/issues/new) diff --git a/doc_requirements.txt b/doc_requirements.txt index 7497dfa3..d7e92631 100644 --- a/doc_requirements.txt +++ b/doc_requirements.txt @@ -4,3 +4,4 @@ setuptools mkdocstrings[python] mike mkdocs-material +mkdocs-awesome-pages-plugin diff --git a/docs/index.md b/docs/index.md index a3097617..612c7a5e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,52 +1 @@ ---- -hide: - - navigation ---- -# PySlurm: Slurm Interface to python - -This module provides a low-level Python wrapper around the Slurm C-API using Cython. - -::: pyslurm.config - handler: python - -::: pyslurm.front_end - handler: python - -::: pyslurm.hostlist - handler: python - -::: pyslurm.job - handler: python - -::: pyslurm.jobstep - handler: python - -::: pyslurm.node - handler: python - -::: pyslurm.partition - handler: python - -::: pyslurm.reservation - handler: python - -::: pyslurm.slurmdb_events - handler: python - -::: pyslurm.slurmdb_reservations - handler: python - -::: pyslurm.slurmdb_clusters - handler: python - -::: pyslurm.slurmdb_jobs - handler: python - -::: pyslurm.statistics - handler: python - -::: pyslurm.topology - handler: python - -::: pyslurm.trigger - handler: python \ No newline at end of file +--8<-- "README.md" diff --git a/docs/logo.png b/docs/logo.png new file mode 120000 index 00000000..a9c1a7c8 --- /dev/null +++ b/docs/logo.png @@ -0,0 +1 @@ +../logo.png \ No newline at end of file diff --git a/docs/reference/.pages b/docs/reference/.pages new file mode 100644 index 00000000..4c3f8599 --- /dev/null +++ b/docs/reference/.pages @@ -0,0 +1,3 @@ +title: API Reference +nav: + - ... diff --git a/docs/reference/config.md b/docs/reference/config.md new file mode 100644 index 00000000..94b0438e --- /dev/null +++ b/docs/reference/config.md @@ -0,0 +1,10 @@ +--- +title: Config +--- + +!!! warning + This API is currently being completely reworked, and is subject to be + removed in the future when a replacement is introduced + +::: pyslurm.config + handler: python diff --git a/docs/reference/db/.pages b/docs/reference/db/.pages new file mode 100644 index 00000000..b7263357 --- /dev/null +++ b/docs/reference/db/.pages @@ -0,0 +1,3 @@ +title: Database +nav: + - ... diff --git a/docs/reference/db/cluster.md b/docs/reference/db/cluster.md new file mode 100644 index 00000000..e6d0a900 --- /dev/null +++ b/docs/reference/db/cluster.md @@ -0,0 +1,10 @@ +--- +title: Cluster +--- + +!!! warning + This API is currently being completely reworked, and is subject to be + removed in the future when a replacement is introduced + +::: pyslurm.slurmdb_clusters + handler: python diff --git a/docs/reference/db/connection.md b/docs/reference/db/connection.md new file mode 100644 index 00000000..27c904fc --- /dev/null +++ b/docs/reference/db/connection.md @@ -0,0 +1,6 @@ +--- +title: Connection +--- + +::: pyslurm.db.Connection + handler: python diff --git a/docs/reference/db/event.md b/docs/reference/db/event.md new file mode 100644 index 00000000..020abcac --- /dev/null +++ b/docs/reference/db/event.md @@ -0,0 +1,10 @@ +--- +title: Event +--- + +!!! warning + This API is currently being completely reworked, and is subject to be + removed in the future when a replacement is introduced + +::: pyslurm.slurmdb_events + handler: python diff --git a/docs/reference/db/index.md b/docs/reference/db/index.md new file mode 100644 index 00000000..98f3b38e --- /dev/null +++ b/docs/reference/db/index.md @@ -0,0 +1,4 @@ +# pyslurm.db + +The `pyslurm.db` package contains all functionality to interact with the Slurm +Database Daemon (slurmdbd) diff --git a/docs/reference/db/job.md b/docs/reference/db/job.md new file mode 100644 index 00000000..a2c7fadd --- /dev/null +++ b/docs/reference/db/job.md @@ -0,0 +1,13 @@ +--- +title: Job +--- + +!!! note + This supersedes the [pyslurm.slurmdb_job](../old/db/job.md) class, which + will be removed in a future release + +::: pyslurm.db.Job + handler: python + +::: pyslurm.db.Jobs + handler: python diff --git a/docs/reference/db/jobsearchfilter.md b/docs/reference/db/jobsearchfilter.md new file mode 100644 index 00000000..fa3864c5 --- /dev/null +++ b/docs/reference/db/jobsearchfilter.md @@ -0,0 +1,6 @@ +--- +title: JobSearchFilter +--- + +::: pyslurm.db.JobSearchFilter + handler: python diff --git a/docs/reference/db/jobstats.md b/docs/reference/db/jobstats.md new file mode 100644 index 00000000..35f31ac6 --- /dev/null +++ b/docs/reference/db/jobstats.md @@ -0,0 +1,6 @@ +--- +title: JobStatistics +--- + +::: pyslurm.db.JobStatistics + handler: python diff --git a/docs/reference/db/jobstep.md b/docs/reference/db/jobstep.md new file mode 100644 index 00000000..392fab65 --- /dev/null +++ b/docs/reference/db/jobstep.md @@ -0,0 +1,9 @@ +--- +title: JobStep +--- + +::: pyslurm.db.JobStep + handler: python + +::: pyslurm.db.JobSteps + handler: python diff --git a/docs/reference/db/reservation.md b/docs/reference/db/reservation.md new file mode 100644 index 00000000..1a1af0c4 --- /dev/null +++ b/docs/reference/db/reservation.md @@ -0,0 +1,10 @@ +--- +title: Reservation +--- + +!!! warning + This API is currently being completely reworked, and is subject to be + removed in the future when a replacement is introduced + +::: pyslurm.slurmdb_reservations + handler: python diff --git a/docs/reference/exceptions.md b/docs/reference/exceptions.md new file mode 100644 index 00000000..90876435 --- /dev/null +++ b/docs/reference/exceptions.md @@ -0,0 +1,9 @@ +--- +title: Exceptions +--- + +::: pyslurm.PyslurmError + handler: python + +::: pyslurm.RPCError + handler: python diff --git a/docs/reference/frontend.md b/docs/reference/frontend.md new file mode 100644 index 00000000..5247e540 --- /dev/null +++ b/docs/reference/frontend.md @@ -0,0 +1,10 @@ +--- +title: Frontend +--- + +!!! warning + This API is currently being completely reworked, and is subject to be + removed in the future when a replacement is introduced + +::: pyslurm.front_end + handler: python diff --git a/docs/reference/hostlist.md b/docs/reference/hostlist.md new file mode 100644 index 00000000..dc2d81ee --- /dev/null +++ b/docs/reference/hostlist.md @@ -0,0 +1,10 @@ +--- +title: Hostlist +--- + +!!! warning + This API is currently being completely reworked, and is subject to be + removed in the future when a replacement is introduced + +::: pyslurm.hostlist + handler: python diff --git a/docs/reference/index.md b/docs/reference/index.md new file mode 100644 index 00000000..e49352fd --- /dev/null +++ b/docs/reference/index.md @@ -0,0 +1,48 @@ +# pyslurm + +The `pyslurm` package is a wrapper around the Slurm C-API + + +!!! warning + Please note that the `pyslurm` API is currently being completely reworked. + Reworked classes and functions that replace functionality of the old API + will be marked as such, with a link to the documentation of its old + counterpart. + + Old API functionality that is already replaced is marked as deprecated, + and will be removed at some point in the future. + + The new reworked classes will be tested thoroughly before making them + available here, although it is of course still possible that some bugs may + appear here and there, which we will try to identify as best as possible! + + In addition, since these classes are pretty new, their interface + (precisely: attribute names, return types) should not yet be considered + 100% stable, and changes may be made in rare cases if it makes sense to do + so. + + If you are using the new-style API, we would like to know your feedback on + it! + + +## Functionality already reworked: + +* Job API + * [pyslurm.Job][] + * [pyslurm.JobStep][] + * [pyslurm.JobSteps][] + * [pyslurm.Jobs][] + * [pyslurm.JobSubmitDescription][] +* Database Job API + * [pyslurm.db.Job][] + * [pyslurm.db.JobStep][] + * [pyslurm.db.Jobs][] + * [pyslurm.db.JobSearchFilter][] +* Node API + * [pyslurm.Node][] + * [pyslurm.Nodes][] +* New Exceptions + * [pyslurm.RPCError][] + * [pyslurm.PyslurmError][] +* New utility functions + * [pyslurm.utils][] diff --git a/docs/reference/job.md b/docs/reference/job.md new file mode 100644 index 00000000..8e3d0c6e --- /dev/null +++ b/docs/reference/job.md @@ -0,0 +1,13 @@ +--- +title: Job +--- + +!!! note + This supersedes the [pyslurm.job](old/job.md) class, which will be + removed in a future release + +::: pyslurm.Job + handler: python + +::: pyslurm.Jobs + handler: python diff --git a/docs/reference/jobstep.md b/docs/reference/jobstep.md new file mode 100644 index 00000000..2ce6ef7f --- /dev/null +++ b/docs/reference/jobstep.md @@ -0,0 +1,13 @@ +--- +title: JobStep +--- + +!!! note + This supersedes the [pyslurm.jobstep](old/jobstep.md) class, which + will be removed in a future release + +::: pyslurm.JobStep + handler: python + +::: pyslurm.JobSteps + handler: python diff --git a/docs/reference/jobsubmitdescription.md b/docs/reference/jobsubmitdescription.md new file mode 100644 index 00000000..bd31bac9 --- /dev/null +++ b/docs/reference/jobsubmitdescription.md @@ -0,0 +1,6 @@ +--- +title: JobSubmitDescription +--- + +::: pyslurm.JobSubmitDescription + handler: python diff --git a/docs/reference/node.md b/docs/reference/node.md new file mode 100644 index 00000000..ccb16c54 --- /dev/null +++ b/docs/reference/node.md @@ -0,0 +1,13 @@ +--- +title: Node +--- + +!!! note + This supersedes the [pyslurm.node](old/node.md) class, which will be + removed in a future release + +::: pyslurm.Node + handler: python + +::: pyslurm.Nodes + handler: python diff --git a/docs/reference/old/.pages b/docs/reference/old/.pages new file mode 100644 index 00000000..ae2a9b18 --- /dev/null +++ b/docs/reference/old/.pages @@ -0,0 +1,3 @@ +hide: true +nav: + - ... diff --git a/docs/reference/old/db/.pages b/docs/reference/old/db/.pages new file mode 100644 index 00000000..ae2a9b18 --- /dev/null +++ b/docs/reference/old/db/.pages @@ -0,0 +1,3 @@ +hide: true +nav: + - ... diff --git a/docs/reference/old/db/job.md b/docs/reference/old/db/job.md new file mode 100644 index 00000000..4046026c --- /dev/null +++ b/docs/reference/old/db/job.md @@ -0,0 +1,10 @@ +--- +title: Job +--- + +!!! warning + This is superseded by [pyslurm.db.Job](../../db/job.md) class and will + be removed in a future release + +::: pyslurm.slurmdb_jobs + handler: python diff --git a/docs/reference/old/job.md b/docs/reference/old/job.md new file mode 100644 index 00000000..fb8f694a --- /dev/null +++ b/docs/reference/old/job.md @@ -0,0 +1,10 @@ +--- +title: Job +--- + +!!! warning + This class is superseded by [pyslurm.Job](../job.md) and will be removed + in a future release. + +::: pyslurm.job + handler: python diff --git a/docs/reference/old/jobstep.md b/docs/reference/old/jobstep.md new file mode 100644 index 00000000..2147e53b --- /dev/null +++ b/docs/reference/old/jobstep.md @@ -0,0 +1,10 @@ +--- +title: JobStep +--- + +!!! warning + This class is superseded by [pyslurm.JobStep](../jobstep.md) and will be + removed in a future release. + +::: pyslurm.jobstep + handler: python diff --git a/docs/reference/old/node.md b/docs/reference/old/node.md new file mode 100644 index 00000000..ec80324a --- /dev/null +++ b/docs/reference/old/node.md @@ -0,0 +1,10 @@ +--- +title: Node +--- + +!!! warning + This class is superseded by [pyslurm.Node](../node.md) and will be + removed in a future release. + +::: pyslurm.node + handler: python diff --git a/docs/reference/partition.md b/docs/reference/partition.md new file mode 100644 index 00000000..6ab4b865 --- /dev/null +++ b/docs/reference/partition.md @@ -0,0 +1,10 @@ +--- +title: Partition +--- + +!!! warning + This API is currently being completely reworked, and is subject to be + removed in the future when a replacement is introduced + +::: pyslurm.partition + handler: python diff --git a/docs/reference/reservation.md b/docs/reference/reservation.md new file mode 100644 index 00000000..563e29db --- /dev/null +++ b/docs/reference/reservation.md @@ -0,0 +1,10 @@ +--- +title: Reservation +--- + +!!! warning + This API is currently being completely reworked, and is subject to be + removed in the future when a replacement is introduced + +::: pyslurm.reservation + handler: python diff --git a/docs/reference/statistics.md b/docs/reference/statistics.md new file mode 100644 index 00000000..1f2b2e37 --- /dev/null +++ b/docs/reference/statistics.md @@ -0,0 +1,10 @@ +--- +title: Statistics +--- + +!!! warning + This API is currently being completely reworked, and is subject to be + removed in the future when a replacement is introduced + +::: pyslurm.statistics + handler: python diff --git a/docs/reference/topology.md b/docs/reference/topology.md new file mode 100644 index 00000000..1cb107a1 --- /dev/null +++ b/docs/reference/topology.md @@ -0,0 +1,10 @@ +--- +title: Topology +--- + +!!! warning + This API is currently being completely reworked, and is subject to be + removed in the future when a replacement is introduced + +::: pyslurm.topology + handler: python diff --git a/docs/reference/trigger.md b/docs/reference/trigger.md new file mode 100644 index 00000000..308a3e3f --- /dev/null +++ b/docs/reference/trigger.md @@ -0,0 +1,10 @@ +--- +title: Trigger +--- + +!!! warning + This API is currently being completely reworked, and is subject to be + removed in the future when a replacement is introduced + +::: pyslurm.trigger + handler: python diff --git a/docs/reference/utilities.md b/docs/reference/utilities.md new file mode 100644 index 00000000..b290d055 --- /dev/null +++ b/docs/reference/utilities.md @@ -0,0 +1,39 @@ +--- +title: Utilities +--- + +::: pyslurm.utils + handler: python + +::: pyslurm.utils.timestr_to_secs + handler: python + +::: pyslurm.utils.timestr_to_mins + handler: python + +::: pyslurm.utils.secs_to_timestr + handler: python + +::: pyslurm.utils.mins_to_timestr + handler: python + +::: pyslurm.utils.date_to_timestamp + handler: python + +::: pyslurm.utils.timestamp_to_date + handler: python + +::: pyslurm.utils.expand_range_str + handler: python + +::: pyslurm.utils.humanize + handler: python + +::: pyslurm.utils.dehumanize + handler: python + +::: pyslurm.utils.nodelist_from_range_str + handler: python + +::: pyslurm.utils.nodelist_to_range_str + handler: python diff --git a/docs/stylesheets/extra.css b/docs/stylesheets/extra.css index 18fe35d6..9562d9be 100644 --- a/docs/stylesheets/extra.css +++ b/docs/stylesheets/extra.css @@ -1,4 +1,4 @@ /* Maximum space for text block */ .md-grid { - max-width: 70%; + max-width: 75%; } diff --git a/docs/pyslurm-docs.png b/logo.png similarity index 100% rename from docs/pyslurm-docs.png rename to logo.png diff --git a/mkdocs.yml b/mkdocs.yml index 56aa68c6..bd62c384 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,17 +1,57 @@ -site_name: pyslurm +site_dir: "site" +site_name: "pyslurm" +site_url: "https://pyslurm.github.io" +repo_url: "https://github.com/PySlurm/pyslurm" +repo_name: "PySlurm/pyslurm" +copyright: Copyright © 2023 The PySlurm Authors + theme: name: "material" - logo: pyslurm-docs.png + logo: logo.png + features: + - navigation.sections + - navigation.indexes + - navigation.tabs + # - navigation.tabs.sticky + - navigation.top + - content.code.copy + - toc.follow + palette: + - media: "(prefers-color-scheme: light)" + scheme: default + accent: purple + toggle: + icon: material/brightness-7 + name: Switch to dark mode + - media: "(prefers-color-scheme: dark)" + scheme: slate + toggle: + icon: material/brightness-4 + name: Switch to light mode + font: + text: Roboto + code: Roboto Mono + plugins: -- search -- mkdocstrings: - handlers: - python: - options: - filters: ["!^_"] - docstring_style: google - show_signature: true - show_root_heading: true + - search + - awesome-pages + - mike + - mkdocstrings: + handlers: + python: + import: + - https://docs.python.org/3/objects.inv + options: + filters: ["!^_"] + docstring_style: google + show_signature: true + show_root_heading: true + +markdown_extensions: + - admonition + - pymdownx.snippets: + check_paths: true + extra: version: provider: mike diff --git a/pyslurm/__init__.py b/pyslurm/__init__.py index aa9e26c6..750199da 100644 --- a/pyslurm/__init__.py +++ b/pyslurm/__init__.py @@ -1,10 +1,6 @@ -""" -PySlurm: Python bindings for the Slurm C API -============================================ - -PySlurm is a Cython wrapper around Slurm C API functions. +"""pyslurm package -More information about Slurm can be found at https://slurm.schedmd.com. +pyslurm is a wrapper around the Slurm C-API. """ from __future__ import absolute_import @@ -16,6 +12,9 @@ from .pyslurm import * from .__version__ import __version__ +from pyslurm import utils +from pyslurm import db + from pyslurm.core.job import ( Job, Jobs, @@ -23,38 +22,12 @@ JobSteps, JobSubmitDescription, ) - -from pyslurm.core import db from pyslurm.core.node import Node, Nodes - -import pyslurm.core.error +from pyslurm.core import error from pyslurm.core.error import ( + PyslurmError, RPCError, ) - -# Utility time functions -from pyslurm.core.common.ctime import ( - timestr_to_secs, - timestr_to_mins, - secs_to_timestr, - mins_to_timestr, - date_to_timestamp, - timestamp_to_date, -) - -# General utility functions -from pyslurm.core.common import ( - uid_to_name, - gid_to_name, - user_to_uid, - group_to_gid, - expand_range_str, - humanize, - dehumanize, - nodelist_from_range_str, - nodelist_to_range_str, -) - from pyslurm.core import slurmctld # Initialize slurm api diff --git a/pyslurm/api.pxd b/pyslurm/api.pxd index 9b19ec9a..b780fdba 100644 --- a/pyslurm/api.pxd +++ b/pyslurm/api.pxd @@ -23,4 +23,4 @@ # cython: language_level=3 from pyslurm cimport slurm -from pyslurm.core.common cimport cstr +from pyslurm.utils cimport cstr diff --git a/pyslurm/core/error.pyx b/pyslurm/core/error.pyx index 69130abd..a5924d08 100644 --- a/pyslurm/core/error.pyx +++ b/pyslurm/core/error.pyx @@ -22,7 +22,7 @@ # cython: c_string_type=unicode, c_string_encoding=default # cython: language_level=3 -from pyslurm.core.common cimport cstr +from pyslurm.utils cimport cstr from pyslurm cimport slurm from pyslurm.slurm cimport slurm_get_errno @@ -65,7 +65,11 @@ def get_last_slurm_error(): return (errno, slurm_strerror(errno)) -class RPCError(Exception): +class PyslurmError(Exception): + """The base Exception for all Pyslurm errors.""" + + +class RPCError(PyslurmError): """Exception for handling Slurm RPC errors. Args: @@ -75,6 +79,13 @@ class RPCError(Exception): msg (str): An optional, custom error description. If this is set, the errno will not be translated to its string representation. + + Examples: + >>> import pyslurm + ... try: + ... myjob = pyslurm.Job.load(9999) + ... except pyslurm.RPCError as e: + ... print("Loading the Job failed") """ def __init__(self, errno=slurm.SLURM_ERROR, msg=None): self.msg = msg diff --git a/pyslurm/core/job/job.pxd b/pyslurm/core/job/job.pxd index c41c8ced..5e8dfd4f 100644 --- a/pyslurm/core/job/job.pxd +++ b/pyslurm/core/job/job.pxd @@ -22,9 +22,9 @@ # cython: c_string_type=unicode, c_string_encoding=default # cython: language_level=3 -from pyslurm.core.common cimport cstr, ctime -from pyslurm.core.common.uint cimport * -from pyslurm.core.common.ctime cimport time_t +from pyslurm.utils cimport cstr, ctime +from pyslurm.utils.uint cimport * +from pyslurm.utils.ctime cimport time_t from libc.string cimport memcpy, memset from libc.stdint cimport uint8_t, uint16_t, uint32_t, uint64_t, int64_t @@ -71,9 +71,9 @@ cdef class Jobs(dict): """A collection of Job objects. Args: - jobs (Union[list, dict], optional): + jobs (Union[list, dict], optional=None): Jobs to initialize this collection with. - freeze (bool, optional): + frozen (bool, optional=False): Control whether this collection is "frozen" when reloading Job information. @@ -89,7 +89,7 @@ cdef class Jobs(dict): Total amount of CPU-Time used by all the Jobs in the collection. This is the result of multiplying the run_time with the amount of cpus for each job. - freeze (bool): + frozen (bool): If this is set to True and the reload() method is called, then *ONLY* Jobs that already exist in this collection will be reloaded. New Jobs that are discovered will not be added to this @@ -103,7 +103,7 @@ cdef class Jobs(dict): slurm_job_info_t tmp_info cdef public: - freeze + frozen cdef class Job: diff --git a/pyslurm/core/job/job.pyx b/pyslurm/core/job/job.pyx index 1e160c80..7f5beae3 100644 --- a/pyslurm/core/job/job.pyx +++ b/pyslurm/core/job/job.pyx @@ -31,16 +31,16 @@ from os import WIFSIGNALED, WIFEXITED, WTERMSIG, WEXITSTATUS import re from typing import Union -from pyslurm.core.common import cstr, ctime -from pyslurm.core.common.uint import * +from pyslurm.utils import cstr, ctime +from pyslurm.utils.uint import * from pyslurm.core.job.util import * from pyslurm.core.error import ( RPCError, verify_rpc, slurm_errno, ) -from pyslurm.core.common.ctime import _raw_time -from pyslurm.core.common import ( +from pyslurm.utils.ctime import _raw_time +from pyslurm.utils.helpers import ( uid_to_name, gid_to_name, signal_to_num, @@ -59,8 +59,8 @@ cdef class Jobs(dict): def __dealloc__(self): slurm_free_job_info_msg(self.info) - def __init__(self, jobs=None, freeze=False): - self.freeze = freeze + def __init__(self, jobs=None, frozen=False): + self.frozen = frozen if isinstance(jobs, dict): self.update(jobs) @@ -72,7 +72,7 @@ cdef class Jobs(dict): self[job.id] = job @staticmethod - def load(preload_passwd_info=False, freeze=False): + def load(preload_passwd_info=False, frozen=False): """Retrieve all Jobs from the Slurm controller Args: @@ -83,11 +83,11 @@ cdef class Jobs(dict): where a UID/GID is translated to a name. If True, the information will fetched and stored in each of the Job instances. - freeze (bool, optional): - Decide whether this collection of Jobs should be "frozen". + frozen (bool, optional): + Decide whether this collection of Jobs should be frozen. Returns: - (Jobs): A collection of Job objects. + (pyslurm.Jobs): A collection of Job objects. Raises: RPCError: When getting all the Jobs from the slurmctld failed. @@ -134,7 +134,7 @@ cdef class Jobs(dict): # instance. jobs.info.record_count = 0 - jobs.freeze = freeze + jobs.frozen = frozen return jobs def reload(self): @@ -149,12 +149,12 @@ cdef class Jobs(dict): if jid in reloaded_jobs: # Put the new data in. self[jid] = reloaded_jobs[jid] - elif not self.freeze: + elif not self.frozen: # Remove this instance from the current collection, as the Job # doesn't exist anymore. del self[jid] - if not self.freeze: + if not self.frozen: for jid in reloaded_jobs: if jid not in self: self[jid] = reloaded_jobs[jid] @@ -167,8 +167,8 @@ cdef class Jobs(dict): This function fills in the "steps" attribute for all Jobs in the collection. - Note: - Pending Jobs will be ignored, since they don't have any Steps yet. + Note: Pending Jobs will be ignored, since they don't have any Steps + yet. Raises: RPCError: When retrieving the Job information for all the Steps @@ -380,8 +380,8 @@ cdef class Job: RPCError: When cancelling the Job was not successful. Examples: - >>> from pyslurm import Job - >>> Job(9999).cancel() + >>> import pyslurm + >>> pyslurm.Job(9999).cancel() """ self.send_signal(9) @@ -394,8 +394,8 @@ cdef class Job: RPCError: When suspending the Job was not successful. Examples: - >>> from pyslurm import Job - >>> Job(9999).suspend() + >>> import pyslurm + >>> pyslurm.Job(9999).suspend() """ # TODO: Report as a misbehaviour to schedmd that slurm_suspend is not # correctly returning error code when it cannot find the job in @@ -413,8 +413,8 @@ cdef class Job: RPCError: When unsuspending the Job was not successful. Examples: - >>> from pyslurm import Jobs - >>> Job(9999).unsuspend() + >>> import pyslurm + >>> pyslurm.Job(9999).unsuspend() """ # Same problem as described in suspend() verify_rpc(slurm_resume(self.id)) @@ -425,7 +425,7 @@ cdef class Job: Implements the slurm_update_job RPC. Args: - changes (JobSubmitDescription): + changes (pyslurm.JobSubmitDescription): A JobSubmitDescription object which contains all the modifications that should be done on the Job. @@ -433,11 +433,11 @@ cdef class Job: RPCError: When updating the Job was not successful. Examples: - >>> from pyslurm import Job, JobSubmitDescription + >>> import pyslurm >>> >>> # Setting the new time-limit to 20 days - >>> changes = JobSubmitDescription(time_limit="20-00:00:00") - >>> Job(9999).modify(changes) + >>> changes = pyslurm.JobSubmitDescription(time_limit="20-00:00:00") + >>> pyslurm.Job(9999).modify(changes) """ changes._create_job_submit_desc(is_update=True) changes.ptr.job_id = self.id @@ -454,20 +454,19 @@ cdef class Job: release the Job again. If you specify the mode as "user", the User will also be able to release the job. - Note: - Uses the modify() function to set the Job's priority to 0. + Note: Uses the modify() function to set the Job's priority to 0. Raises: RPCError: When holding the Job was not successful. Examples: - >>> from pyslurm import Job + >>> import pyslurm >>> >>> # Holding a Job (in "admin" mode by default) - >>> Job(9999).hold() + >>> pyslurm.Job(9999).hold() >>> >>> # Holding a Job in "user" mode - >>> Job(9999).hold(mode="user") + >>> pyslurm.Job(9999).hold(mode="user") """ cdef JobSubmitDescription job_sub = JobSubmitDescription(priority=0) @@ -479,16 +478,15 @@ cdef class Job: def release(self): """Release a currently held Job, allowing it to be scheduled again. - Note: - Uses the modify() function to reset the priority back to + Note: Uses the modify() function to reset the priority back to be controlled by the slurmctld's priority calculation routine. Raises: RPCError: When releasing a held Job was not successful. Examples: - >>> from pyslurm import Job - >>> Job(9999).release() + >>> import pyslurm + >>> pyslurm.Job(9999).release() """ self.modify(JobSubmitDescription(priority=slurm.INFINITE)) @@ -498,7 +496,7 @@ cdef class Job: Implements the slurm_requeue RPC. Args: - hold (bool): + hold (bool, optional): Controls whether the Job should be put in a held state or not. Default for this is 'False', so it will not be held. @@ -506,14 +504,14 @@ cdef class Job: RPCError: When requeing the Job was not successful. Examples: - >>> from pyslurm import Job + >>> import pyslurm >>> >>> # Requeing a Job while allowing it to be >>> # scheduled again immediately - >>> Job(9999).requeue() + >>> pyslurm.Job(9999).requeue() >>> >>> # Requeing a Job while putting it in a held state - >>> Job(9999).requeue(hold=True) + >>> pyslurm.Job(9999).requeue(hold=True) """ cdef uint32_t flags = 0 @@ -535,28 +533,24 @@ cdef class Job: RPCError: When sending the message to the Job was not successful. Examples: - >>> from pyslurm import Job - >>> Job(9999).notify("Hello Friends!") + >>> import pyslurm + >>> pyslurm.Job(9999).notify("Hello Friends!") """ verify_rpc(slurm_notify_job(self.id, msg)) def get_batch_script(self): """Return the content of the script for a Batch-Job. - Note: - The string returned also includes all the "\n" characters - (new-line). - Returns: (str): The content of the batch script. Raises: RPCError: When retrieving the Batch-Script for the Job was not - successful. + successful. Examples: - >>> from pyslurm import Job - >>> script = Job(9999).get_batch_script() + >>> import pyslurm + >>> script = pyslurm.Job(9999).get_batch_script() """ # The code for this function was taken from here: # https://github.com/SchedMD/slurm/blob/7162f15af8deaf02c3bbf940d59e818cdeb5c69d/src/api/job_info.c#L1319 @@ -1229,13 +1223,14 @@ cdef class Job: def get_resource_layout_per_node(self): """Retrieve the resource layout of this Job on each node. - This contains the following information: + The dict returned contains the following information for each node: * cpu_ids (str) * gres (dict) * memory (int) Returns: - (dict): Resource layout + (dict): Resource layout, where the key is the name of the name and + its value another dict with the components described above. """ # The code for this function is a modified reimplementation from here: # https://github.com/SchedMD/slurm/blob/d525b6872a106d32916b33a8738f12510ec7cf04/src/api/job_info.c#L739 diff --git a/pyslurm/core/job/sbatch_opts.pyx b/pyslurm/core/job/sbatch_opts.pyx index 91724d29..c6e0b400 100644 --- a/pyslurm/core/job/sbatch_opts.pyx +++ b/pyslurm/core/job/sbatch_opts.pyx @@ -184,8 +184,8 @@ def _find_opt(opt): def _parse_opts_from_batch_script(desc, script, overwrite): flags_and_vals = {} - if not script or not Path(script).is_file(): - return None + if not Path(script).is_file(): + raise ValueError("The script path you provided is not valid.") script = Path(script).read_text() for line in script.splitlines(): diff --git a/pyslurm/core/job/step.pxd b/pyslurm/core/job/step.pxd index 4cdd6c49..ed61d4d9 100644 --- a/pyslurm/core/job/step.pxd +++ b/pyslurm/core/job/step.pxd @@ -24,7 +24,7 @@ from libc.stdint cimport uint8_t, uint16_t, uint32_t, uint64_t from .job cimport Job - +from libc.string cimport memcpy, memset from pyslurm cimport slurm from pyslurm.slurm cimport ( job_step_info_t, @@ -43,9 +43,14 @@ from pyslurm.slurm cimport ( xfree, try_xmalloc, ) +from pyslurm.utils cimport cstr, ctime +from pyslurm.utils.uint cimport * +from pyslurm.utils.ctime cimport time_t +from pyslurm.core.job.task_dist cimport TaskDistribution + cdef class JobSteps(dict): - """A collection of :obj:`JobStep` objects for a given Job. + """A collection of [`pyslurm.JobStep`][] objects for a given Job. Args: job (Union[Job, int]): @@ -70,11 +75,15 @@ cdef class JobStep: """A Slurm Jobstep Args: - job (Union[Job, int]): + job_id (Union[Job, int], optional=0): The Job this Step belongs to. - step (Union[int, str]): + step_id (Union[int, str], optional=0): Step-ID for this JobStep object. + Other Parameters: + time_limit (int): + Time limit in Minutes for this step. + Raises: MemoryError: If malloc fails to allocate memory. diff --git a/pyslurm/core/job/step.pyx b/pyslurm/core/job/step.pyx index d84330b1..4b05aa5b 100644 --- a/pyslurm/core/job/step.pyx +++ b/pyslurm/core/job/step.pyx @@ -22,22 +22,17 @@ # cython: c_string_type=unicode, c_string_encoding=default # cython: language_level=3 -from libc.string cimport memcpy, memset -from pyslurm.core.common cimport cstr, ctime -from pyslurm.core.common import cstr, ctime -from pyslurm.core.common.uint cimport * -from pyslurm.core.common.uint import * -from pyslurm.core.common.ctime cimport time_t +from typing import Union +from pyslurm.utils import cstr, ctime +from pyslurm.utils.uint import * from pyslurm.core.error import RPCError, verify_rpc -from pyslurm.core.common import ( +from pyslurm.utils.helpers import ( signal_to_num, instance_to_dict, uid_to_name, ) from pyslurm.core.job.util import cpu_freq_int_to_str -from pyslurm.core.job.task_dist cimport TaskDistribution - -from pyslurm.core.common.ctime import ( +from pyslurm.utils.ctime import ( secs_to_timestr, mins_to_timestr, timestr_to_mins, @@ -59,6 +54,15 @@ cdef class JobSteps(dict): @staticmethod def load(job): + """Load the Steps for a specific Job + + Args: + job (Union[Job, int]): + The Job for which the Steps should be loaded + + Returns: + (pyslurm.JobSteps): JobSteps of the Job + """ cdef Job _job _job = Job.load(job.id) if isinstance(job, Job) else Job.load(job) return JobSteps._load(_job) @@ -187,7 +191,7 @@ cdef class JobStep: Implements the slurm_get_job_steps RPC. Args: - job_id (Union[Job, int]): + job_id (Union[pyslurm.Job, int]): ID of the Job the Step belongs to. step_id (Union[int, str]): Step-ID for the Step to be loaded. @@ -249,12 +253,12 @@ cdef class JobStep: Examples: Specifying the signal as a string: - >>> from pyslurm import JobStep - >>> JobStep(9999, 1).send_signal("SIGUSR1") + >>> import pyslurm + >>> pyslurm.JobStep(9999, 1).send_signal("SIGUSR1") or passing in a numeric signal: - >>> JobStep(9999, 1).send_signal(9) + >>> pyslurm.JobStep(9999, 1).send_signal(9) """ step_id = self.ptr.step_id.step_id sig = signal_to_num(signal) @@ -269,54 +273,37 @@ cdef class JobStep: RPCError: When cancelling the Job was not successful. Examples: - >>> from pyslurm import JobStep - >>> JobStep(9999, 1).cancel() + >>> import pyslurm + >>> pyslurm.JobStep(9999, 1).cancel() """ step_id = self.ptr.step_id.step_id verify_rpc(slurm_kill_job_step(self.job_id, step_id, 9)) - def modify(self, step=None, **kwargs): + def modify(self, changes): """Modify a job step. Implements the slurm_update_step RPC. Args: - step (JobStep): + changes (pyslurm.JobStep): Another JobStep object which contains all the changes that should be applied to this instance. - **kwargs: - You can also specify all the changes as keyword arguments. - Allowed values are only attributes which can actually be set - on a JobStep instance. If a step is explicitly specified as - parameter, all **kwargs will be ignored. - Raises: RPCError: When updating the JobStep was not successful. Examples: - >>> from pyslurm import JobStep + >>> import pyslurm >>> >>> # Setting the new time-limit to 20 days - >>> changes = JobStep(time_limit="20-00:00:00") - >>> JobStep(9999, 1).modify(changes) - >>> - >>> # Or by specifying the changes directly to the modify function - >>> JobStep(9999, 1).modify(time_limit="20-00:00:00") + >>> changes = pyslurm.JobStep(time_limit="20-00:00:00") + >>> pyslurm.JobStep(9999, 1).modify(changes) """ - cdef JobStep js = self - - # Allow the user to both specify changes via object and **kwargs. - if step and isinstance(step, JobStep): - js = step - elif kwargs: - js = JobStep(**kwargs) - + cdef JobStep js = changes js._alloc_umsg() js.umsg.step_id = self.ptr.step_id.step_id js.umsg.job_id = self.ptr.step_id.job_id verify_rpc(slurm_update_step(js.umsg)) - def as_dict(self): """JobStep information formatted as a dictionary. @@ -450,6 +437,7 @@ def humanize_step_id(sid): else: return sid + def dehumanize_step_id(sid): if sid == "batch": return slurm.SLURM_BATCH_SCRIPT diff --git a/pyslurm/core/job/submission.pxd b/pyslurm/core/job/submission.pxd index ebf0b0c5..f9c9d877 100644 --- a/pyslurm/core/job/submission.pxd +++ b/pyslurm/core/job/submission.pxd @@ -38,10 +38,18 @@ from pyslurm.slurm cimport ( xfree, try_xmalloc, ) +from pyslurm.utils cimport cstr, ctime +from pyslurm.utils.uint cimport * +from pyslurm.utils.ctime cimport time_t +from pyslurm.core.job.task_dist cimport TaskDistribution cdef class JobSubmitDescription: - """Description of a Slurm Job. + """Submit Description for a Slurm Job. + + Args: + **kwargs (Any, optional=None): + Any valid Attribute this object has Attributes: name (str): diff --git a/pyslurm/core/job/submission.pyx b/pyslurm/core/job/submission.pyx index e1f4039d..f023f035 100644 --- a/pyslurm/core/job/submission.pyx +++ b/pyslurm/core/job/submission.pyx @@ -25,18 +25,15 @@ from os import getcwd from os import environ as pyenviron import re -import typing +from typing import Union, Any import shlex from pathlib import Path -from pyslurm.core.common cimport cstr, ctime -from pyslurm.core.common import cstr -from pyslurm.core.common.uint cimport * -from pyslurm.core.common.uint import * -from pyslurm.core.common.ctime cimport time_t +from pyslurm.utils import cstr +from pyslurm.utils.uint import * from pyslurm.core.job.util import * from pyslurm.core.error import RPCError, verify_rpc from pyslurm.core.job.sbatch_opts import _parse_opts_from_batch_script -from pyslurm.core.common.ctime import ( +from pyslurm.utils.ctime import ( secs_to_timestr, timestr_to_secs, mins_to_timestr, @@ -44,9 +41,7 @@ from pyslurm.core.common.ctime import ( timestamp_to_date, date_to_timestamp, ) -from pyslurm.core.job.task_dist cimport TaskDistribution - -from pyslurm.core.common import ( +from pyslurm.utils.helpers import ( humanize, dehumanize, signal_to_num, @@ -89,10 +84,11 @@ cdef class JobSubmitDescription: MemoryError: If malloc failed to allocate enough memory. Examples: - >>> desc = JobSubmitDescription( - >>> name="test-job", - >>> cpus_per_task=1, - >>> time_limit="10-00:00:00") + >>> import pyslurm + >>> desc = pyslurm.JobSubmitDescription( + ... name="test-job", + ... cpus_per_task=1, + ... time_limit="10-00:00:00") >>> >>> job_id = desc.submit() """ @@ -112,7 +108,8 @@ cdef class JobSubmitDescription: Args: overwrite (bool): If set to True, the value from an option found in the - environment will override its current value. Default is False + environment will override the current value of the attribute + in this instance. Default is False """ self._parse_env(overwrite) @@ -122,8 +119,11 @@ cdef class JobSubmitDescription: Args: overwrite (bool): If set to True, the value from an option found in the in the - batch script will override its current value. Default is False + batch script will override the current value of the attribute + in this instance. Default is False """ + if not self.script: + raise ValueError("You need to set the 'script' attribute first.") _parse_opts_from_batch_script(self, self.script, overwrite) def _parse_env(self, overwrite=False): diff --git a/pyslurm/core/job/task_dist.pxd b/pyslurm/core/job/task_dist.pxd index 5fe76488..4f8a073d 100644 --- a/pyslurm/core/job/task_dist.pxd +++ b/pyslurm/core/job/task_dist.pxd @@ -23,7 +23,7 @@ # cython: language_level=3 from pyslurm cimport slurm -from pyslurm.core.common.uint cimport u16 +from pyslurm.utils.uint cimport u16 from pyslurm.slurm cimport ( task_dist_states_t, ) diff --git a/pyslurm/core/job/util.pyx b/pyslurm/core/job/util.pyx index 7b463b2c..31a31638 100644 --- a/pyslurm/core/job/util.pyx +++ b/pyslurm/core/job/util.pyx @@ -24,8 +24,8 @@ from libc.stdint cimport uint8_t, uint16_t, uint32_t, uint64_t from pyslurm cimport slurm -from pyslurm.core.common.uint import * -from pyslurm.core.common.uint cimport * +from pyslurm.utils.uint import * +from pyslurm.utils.uint cimport * # Note: Maybe consider using libslurmfull again to avoid having to reimplement # some of these functions and keeping track for changes in new releases. diff --git a/pyslurm/core/node.pxd b/pyslurm/core/node.pxd index 3f39ece7..412a290c 100644 --- a/pyslurm/core/node.pxd +++ b/pyslurm/core/node.pxd @@ -24,6 +24,7 @@ from libc.string cimport memcpy, memset from pyslurm cimport slurm +from libc.stdint cimport uint8_t, uint16_t, uint32_t, uint64_t from pyslurm.slurm cimport ( node_info_t, node_info_msg_t, @@ -47,14 +48,20 @@ from pyslurm.slurm cimport ( slurm_node_state_string_complete, slurm_node_state_string, cpu_bind_type_t, + xfree, + try_xmalloc, ) +from pyslurm.utils cimport cstr +from pyslurm.utils cimport ctime +from pyslurm.utils.ctime cimport time_t +from pyslurm.utils.uint cimport * cdef class Nodes(dict): """A collection of Node objects. Args: - nodes (Union[list, dict, str], optional): + nodes (Union[list, dict, str], optional=None): Nodes to initialize this collection with. Attributes: @@ -90,21 +97,30 @@ cdef class Node: """A Slurm node. Args: - name (str): + name (str, optional=None): Name of a node - **kwargs: - Any writable property. Writable attributes include: - * name - * configured_gres - * address - * hostname - * extra - * comment - * weight - * available_features - * active_features - * cpu_binding - * state + + Other Parameters: + configured_gres (dict): + Configured GRES for the node + address (str): + Address of the node + hostname (str): + Hostname of the node + extra (str): + Arbitrary extra string + comment (str): + Comment for the node + weight (int): + Weight associated to the node + available_features (list): + Available features for the node + active_features (list): + Active features for the node + cpu_binding (str): + Default CPU-Binding for the node + state (str): + State of the node Attributes: name (str): diff --git a/pyslurm/core/node.pyx b/pyslurm/core/node.pyx index 17429ce1..050f6262 100644 --- a/pyslurm/core/node.pyx +++ b/pyslurm/core/node.pyx @@ -22,18 +22,13 @@ # cython: c_string_type=unicode, c_string_encoding=default # cython: language_level=3 -from pyslurm.slurm cimport xfree, try_xmalloc -from libc.stdint cimport uint8_t, uint16_t, uint32_t, uint64_t -from pyslurm.core.common cimport cstr -from pyslurm.core.common import cstr -from pyslurm.core.common cimport ctime -from pyslurm.core.common import ctime -from pyslurm.core.common.ctime cimport time_t -from pyslurm.core.common.uint cimport * -from pyslurm.core.common.uint import * +from typing import Union +from pyslurm.utils import cstr +from pyslurm.utils import ctime +from pyslurm.utils.uint import * from pyslurm.core.error import RPCError, verify_rpc -from pyslurm.core.common.ctime import timestamp_to_date, _raw_time -from pyslurm.core.common import ( +from pyslurm.utils.ctime import timestamp_to_date, _raw_time +from pyslurm.utils.helpers import ( uid_to_name, gid_to_name, humanize, @@ -83,7 +78,7 @@ cdef class Nodes(dict): the Node instances. The default is False. Returns: - (Nodes): Collection of node objects. + (pyslurm.Nodes): Collection of node objects. Raises: RPCError: When getting all the Nodes from the slurmctld failed. @@ -136,8 +131,7 @@ cdef class Nodes(dict): def reload(self): """Reload the information for nodes in a collection. - Note: - Only information for nodes which are already in the collection at + Note: Only information for nodes which are already in the collection at the time of calling this method will be reloaded. Raises: @@ -318,21 +312,21 @@ cdef class Node: Implements the slurm_create_node RPC. Args: - future (str, optional): + state (str, optional): An optional state the created Node should have. Allowed values are "future" and "cloud". "future" is the default. Returns: - (Node): This function returns the current Node-instance object - itself. + (pyslurm.Node): This function returns the current Node-instance + object itself. Raises: RPCError: If creating the Node was not successful. MemoryError: If malloc failed to allocate memory. Examples: - >>> from pyslurm import Node - >>> node = Node("testnode").create() + >>> import pyslurm + >>> node = pyslurm.Node("testnode").create() """ if not self.name: raise ValueError("You need to set a node name first.") @@ -344,43 +338,28 @@ cdef class Node: return self - def modify(self, node=None, **kwargs): + def modify(self, changes): """Modify a node. Implements the slurm_update_node RPC. Args: - node (pyslurm.Node): + changes (pyslurm.Node): Another Node object which contains all the changes that should be applied to this instance. - **kwargs: - You can also specify all the changes as keyword arguments. - Allowed values are only attributes which can actually be set - on a Node instance. If a node is explicitly specified as - parameter, all **kwargs will be ignored. Raises: RPCError: When updating the Node was not successful. Examples: - >>> from pyslurm import Node - >>> - >>> # Setting a new weight for the Node - >>> changes = Node(weight=100) - >>> Node("localhost").modify(changes) + >>> import pyslurm >>> - >>> # Or by specifying the changes directly to the modify function - >>> Node("localhost").modify(weight=100) + >>> mynode = pyslurm.Node("localhost") + >>> changes = pyslurm.Node(weight=100) + >>> # Setting the weight to 100 for the "localhost" node + >>> mynode.modify(changes) """ - cdef Node n = self - - # Allow the user to both specify changes via a Node instance or - # **kwargs. - if node and isinstance(node, Node): - n = node - elif kwargs: - n = Node(**kwargs) - + cdef Node n = changes n._alloc_umsg() cstr.fmalloc(&n.umsg.node_names, self.name) verify_rpc(slurm_update_node(n.umsg)) @@ -395,8 +374,8 @@ cdef class Node: MemoryError: If malloc failed to allocate memory. Examples: - >>> from pyslurm import Node - >>> Node("localhost").delete() + >>> import pyslurm + >>> pyslurm.Node("localhost").delete() """ self._alloc_umsg() verify_rpc(slurm_delete_node(self.umsg)) @@ -406,6 +385,11 @@ cdef class Node: Returns: (dict): Node information as dict + + Examples: + >>> import pyslurm + >>> mynode = pyslurm.Node.load("mynode") + >>> mynode_dict = mynode.as_dict() """ return instance_to_dict(self) diff --git a/pyslurm/core/slurmctld.pxd b/pyslurm/core/slurmctld.pxd index f65655c8..0f42fffb 100644 --- a/pyslurm/core/slurmctld.pxd +++ b/pyslurm/core/slurmctld.pxd @@ -29,9 +29,9 @@ from pyslurm.slurm cimport ( slurm_free_ctl_conf, try_xmalloc, ) -from pyslurm.core.common cimport cstr +from pyslurm.utils cimport cstr from libc.stdint cimport uint8_t, uint16_t, uint32_t, uint64_t, int64_t -from pyslurm.core.common.uint cimport * +from pyslurm.utils.uint cimport * cdef class Config: diff --git a/pyslurm/core/db/__init__.pxd b/pyslurm/db/__init__.pxd similarity index 100% rename from pyslurm/core/db/__init__.pxd rename to pyslurm/db/__init__.pxd diff --git a/pyslurm/core/db/__init__.py b/pyslurm/db/__init__.py similarity index 81% rename from pyslurm/core/db/__init__.py rename to pyslurm/db/__init__.py index a742f72b..bb34e232 100644 --- a/pyslurm/core/db/__init__.py +++ b/pyslurm/db/__init__.py @@ -1,5 +1,5 @@ ######################################################################### -# db/__init__.py - database package __init__ file +# db/__init__.py - pyslurm database api ######################################################################### # Copyright (C) 2023 Toni Harzendorf # @@ -19,18 +19,19 @@ # with PySlurm; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -from pyslurm.core.db.connection import Connection -from pyslurm.core.db.step import JobStep -from pyslurm.core.db.job import ( +from .connection import Connection +from .step import JobStep, JobSteps +from .stats import JobStatistics +from .job import ( Job, Jobs, JobSearchFilter, ) -from pyslurm.core.db.tres import ( +from .tres import ( TrackableResource, TrackableResources, ) -from pyslurm.core.db.qos import ( +from .qos import ( QualitiesOfService, QualityOfService, QualityOfServiceSearchFilter, diff --git a/pyslurm/core/db/connection.pxd b/pyslurm/db/connection.pxd similarity index 100% rename from pyslurm/core/db/connection.pxd rename to pyslurm/db/connection.pxd diff --git a/pyslurm/core/db/connection.pyx b/pyslurm/db/connection.pyx similarity index 86% rename from pyslurm/core/db/connection.pyx rename to pyslurm/db/connection.pyx index ff32dd92..eab6572d 100644 --- a/pyslurm/core/db/connection.pyx +++ b/pyslurm/db/connection.pyx @@ -46,7 +46,11 @@ cdef class Connection: RPCError: When opening the connection fails Returns: - (Connection): Connection to slurmdbd + (pyslurm.db.Connection): Connection to slurmdbd + + Examples: + >>> import pyslurm + >>> connection = pyslurm.db.Connection.open() """ cdef Connection conn = Connection.__new__(Connection) conn.ptr = slurmdb_connection_get(&conn.flags) @@ -56,7 +60,14 @@ cdef class Connection: return conn def close(self): - """Close the current connection.""" + """Close the current connection. + + Examples: + >>> import pyslurm + >>> connection = pyslurm.db.Connection.open() + >>> ... + >>> connection.close() + """ if self.is_open: slurmdb_connection_close(&self.ptr) self.ptr = NULL diff --git a/pyslurm/core/db/job.pxd b/pyslurm/db/job.pxd similarity index 94% rename from pyslurm/core/db/job.pxd rename to pyslurm/db/job.pxd index 2b220a05..08673682 100644 --- a/pyslurm/core/db/job.pxd +++ b/pyslurm/db/job.pxd @@ -40,23 +40,24 @@ from pyslurm.slurm cimport ( slurm_job_state_string, slurm_job_reason_string, ) -from pyslurm.core.db.util cimport ( +from pyslurm.db.util cimport ( SlurmList, SlurmListItem, make_char_list, ) -from pyslurm.core.db.step cimport JobStep, JobSteps -from pyslurm.core.db.stats cimport JobStats -from pyslurm.core.db.connection cimport Connection -from pyslurm.core.common cimport cstr -from pyslurm.core.db.qos cimport QualitiesOfService +from pyslurm.db.step cimport JobStep, JobSteps +from pyslurm.db.stats cimport JobStatistics +from pyslurm.db.connection cimport Connection +from pyslurm.utils cimport cstr +from pyslurm.db.qos cimport QualitiesOfService +from pyslurm.db.tres cimport TrackableResources, TrackableResource cdef class JobSearchFilter: """Search conditions for Slurm database Jobs. Args: - **kwargs: + **kwargs (Any, optional=None): Any valid attribute of the object. Attributes: @@ -148,7 +149,7 @@ cdef class JobSearchFilter: cdef class Jobs(dict): - """A collection of Database Jobs.""" + """A collection of [`pyslurm.db.Job`][] objects.""" cdef: SlurmList info Connection db_conn @@ -158,7 +159,7 @@ cdef class Job: """A Slurm Database Job. Args: - job_id (int): + job_id (int, optional=0): An Integer representing a Job-ID. Raises: @@ -167,7 +168,7 @@ cdef class Job: Attributes: steps (pyslurm.db.JobSteps): Steps this Job has - stats (pyslurm.db.JobStats): + stats (pyslurm.db.JobStatistics): Utilization statistics of this Job account (str): Account of the Job. @@ -273,7 +274,7 @@ cdef class Job: cdef public: JobSteps steps - JobStats stats + JobStatistics stats @staticmethod cdef Job from_ptr(slurmdb_job_rec_t *in_ptr) diff --git a/pyslurm/core/db/job.pyx b/pyslurm/db/job.pyx similarity index 92% rename from pyslurm/core/db/job.pyx rename to pyslurm/db/job.pyx index d66f789e..50fd0c8c 100644 --- a/pyslurm/core/db/job.pyx +++ b/pyslurm/db/job.pyx @@ -23,16 +23,17 @@ # cython: language_level=3 from os import WIFSIGNALED, WIFEXITED, WTERMSIG, WEXITSTATUS +from typing import Union from pyslurm.core.error import RPCError -from pyslurm.core.db.tres cimport TrackableResources, TrackableResource from pyslurm.core import slurmctld -from pyslurm.core.common.uint import * -from pyslurm.core.common.ctime import ( +from typing import Any +from pyslurm.utils.uint import * +from pyslurm.utils.ctime import ( date_to_timestamp, timestr_to_mins, _raw_time, ) -from pyslurm.core.common import ( +from pyslurm.utils.helpers import ( gid_to_name, group_to_gid, user_to_uid, @@ -192,7 +193,7 @@ cdef class JobSearchFilter: cdef class Jobs(dict): - def __init__(self, *args, **kwargs): + def __init__(self): # TODO: ability to initialize with existing job objects pass @@ -210,6 +211,21 @@ cdef class Jobs(dict): Raises: RPCError: When getting the Jobs from the Database was not sucessful + + Examples: + Without a Filter the default behaviour applies, which is + simply retrieving all Jobs from the same day: + + >>> import pyslurm + >>> db_jobs = pyslurm.db.Jobs.load() + + Now with a Job Filter, so only Jobs that have specific Accounts + are returned: + + >>> import pyslurm + >>> accounts = ["acc1", "acc2"] + >>> search_filter = pyslurm.db.JobSearchFilter(accounts=accounts) + >>> db_jobs = pyslurm.db.Jobs.load(search_filter) """ cdef: Jobs jobs = Jobs() @@ -245,7 +261,7 @@ cdef class Jobs(dict): job = Job.from_ptr(job_ptr.data) job.qos_data = qos_data job._create_steps() - JobStats._sum_step_stats_for_job(job, job.steps) + JobStatistics._sum_step_stats_for_job(job, job.steps) jobs[job.id] = job return jobs @@ -256,7 +272,7 @@ cdef class Job: def __cinit__(self): self.ptr = NULL - def __init__(self, job_id): + def __init__(self, job_id=0): self._alloc_impl() self.ptr.jobid = int(job_id) @@ -279,7 +295,7 @@ cdef class Job: cdef Job wrap = Job.__new__(Job) wrap.ptr = in_ptr wrap.steps = JobSteps.__new__(JobSteps) - wrap.stats = JobStats() + wrap.stats = JobStatistics() return wrap @staticmethod @@ -291,11 +307,24 @@ cdef class Job: ID of the Job to be loaded. Returns: - (pyslurm.db.Job): Returns a new Job instance + (pyslurm.Job): Returns a new Database Job instance Raises: RPCError: If requesting the information for the database Job was not sucessful. + + Examples: + >>> import pyslurm + >>> db_job = pyslurm.db.Job.load(10000) + + In the above example, attribute like "script" and "environment" + are not populated. You must explicitly request one of them to be + loaded: + + >>> import pyslurm + >>> db_job = pyslurm.db.Job.load(10000, with_script=True) + >>> print(db_job.script) + """ jfilter = JobSearchFilter(ids=[int(job_id)], with_script=with_script, with_env=with_env) @@ -321,6 +350,11 @@ cdef class Job: Returns: (dict): Database Job information as dict + + Examples: + >>> import pyslurm + >>> myjob = pyslurm.db.Job.load(10000) + >>> myjob_dict = myjob.as_dict() """ cdef dict out = instance_to_dict(self) diff --git a/pyslurm/core/db/qos.pxd b/pyslurm/db/qos.pxd similarity index 93% rename from pyslurm/core/db/qos.pxd rename to pyslurm/db/qos.pxd index 3ba59dc6..b2b0bcf9 100644 --- a/pyslurm/core/db/qos.pxd +++ b/pyslurm/db/qos.pxd @@ -32,13 +32,13 @@ from pyslurm.slurm cimport ( slurm_preempt_mode_num, try_xmalloc, ) -from pyslurm.core.db.util cimport ( +from pyslurm.db.util cimport ( SlurmList, SlurmListItem, make_char_list, ) -from pyslurm.core.db.connection cimport Connection -from pyslurm.core.common cimport cstr +from pyslurm.db.connection cimport Connection +from pyslurm.utils cimport cstr cdef class QualitiesOfService(dict): diff --git a/pyslurm/core/db/qos.pyx b/pyslurm/db/qos.pyx similarity index 97% rename from pyslurm/core/db/qos.pyx rename to pyslurm/db/qos.pyx index bd5a35de..2851587e 100644 --- a/pyslurm/core/db/qos.pyx +++ b/pyslurm/db/qos.pyx @@ -23,9 +23,7 @@ # cython: language_level=3 from pyslurm.core.error import RPCError -from pyslurm.core.common import ( - instance_to_dict, -) +from pyslurm.utils.helpers import instance_to_dict cdef class QualitiesOfService(dict): @@ -163,7 +161,7 @@ cdef class QualityOfService: Name of the Quality of Service to be loaded. Returns: - (pyslurm.db.QualityOfService): Returns a new QualityOfService + (QualityOfService): Returns a new QualityOfService instance. Raises: diff --git a/pyslurm/core/db/stats.pxd b/pyslurm/db/stats.pxd similarity index 95% rename from pyslurm/core/db/stats.pxd rename to pyslurm/db/stats.pxd index 1f321ab2..682143e6 100644 --- a/pyslurm/core/db/stats.pxd +++ b/pyslurm/db/stats.pxd @@ -28,13 +28,13 @@ from pyslurm.slurm cimport ( slurmdb_stats_t, slurmdb_job_rec_t, ) -from pyslurm.core.db.tres cimport TrackableResources -from pyslurm.core.db.step cimport JobStep, JobSteps -from pyslurm.core.db.job cimport Job -from pyslurm.core.common cimport cstr +from pyslurm.db.tres cimport TrackableResources +from pyslurm.db.step cimport JobStep, JobSteps +from pyslurm.db.job cimport Job +from pyslurm.utils cimport cstr -cdef class JobStats: +cdef class JobStatistics: """Statistics for a Slurm Job or Step. Note: @@ -139,5 +139,5 @@ cdef class JobStats: system_cpu_time @staticmethod - cdef JobStats from_step(JobStep step) + cdef JobStatistics from_step(JobStep step) diff --git a/pyslurm/core/db/stats.pyx b/pyslurm/db/stats.pyx similarity index 97% rename from pyslurm/core/db/stats.pyx rename to pyslurm/db/stats.pyx index bd6606a0..3ae0c8b5 100644 --- a/pyslurm/core/db/stats.pyx +++ b/pyslurm/db/stats.pyx @@ -22,13 +22,13 @@ # cython: c_string_type=unicode, c_string_encoding=default # cython: language_level=3 -from pyslurm.core.common import ( +from pyslurm.utils.helpers import ( nodelist_from_range_str, instance_to_dict, ) -cdef class JobStats: +cdef class JobStatistics: def __init__(self): for attr, val in instance_to_dict(self).items(): @@ -51,8 +51,8 @@ cdef class JobStats: return instance_to_dict(self) @staticmethod - cdef JobStats from_step(JobStep step): - cdef JobStats wrap = JobStats() + cdef JobStatistics from_step(JobStep step): + cdef JobStatistics wrap = JobStatistics() if not &step.ptr.stats: return wrap @@ -143,8 +143,8 @@ cdef class JobStats: @staticmethod def _sum_step_stats_for_job(Job job, JobSteps steps): cdef: - JobStats job_stats = job.stats - JobStats step_stats = None + JobStatistics job_stats = job.stats + JobStatistics step_stats = None for step in steps.values(): step_stats = step.stats diff --git a/pyslurm/core/db/step.pxd b/pyslurm/db/step.pxd similarity index 88% rename from pyslurm/core/db/step.pxd rename to pyslurm/db/step.pxd index 77d45cd2..d13e3da7 100644 --- a/pyslurm/core/db/step.pxd +++ b/pyslurm/db/step.pxd @@ -36,21 +36,23 @@ from pyslurm.slurm cimport ( slurm_job_state_string, slurm_job_reason_string, ) -from pyslurm.core.db.util cimport SlurmList, SlurmListItem -from pyslurm.core.db.connection cimport Connection -from pyslurm.core.common cimport cstr -from pyslurm.core.db.stats cimport JobStats +from pyslurm.db.util cimport SlurmList, SlurmListItem +from pyslurm.db.connection cimport Connection +from pyslurm.utils cimport cstr +from pyslurm.db.stats cimport JobStatistics +from pyslurm.db.tres cimport TrackableResources, TrackableResource cdef class JobSteps(dict): + """A collection of [`pyslurm.db.JobStep`][] objects""" pass cdef class JobStep: - """A Slurm Database Job-step. + """A Slurm Database JobStep. Attributes: - stats (pyslurm.db.JobStats): + stats (pyslurm.db.JobStatistics): Utilization statistics for this Step num_nodes (int): Amount of nodes this Step has allocated @@ -94,7 +96,7 @@ cdef class JobStep: Amount of seconds the Step was suspended """ cdef slurmdb_step_rec_t *ptr - cdef public JobStats stats + cdef public JobStatistics stats @staticmethod cdef JobStep from_ptr(slurmdb_step_rec_t *step) diff --git a/pyslurm/core/db/step.pyx b/pyslurm/db/step.pyx similarity index 95% rename from pyslurm/core/db/step.pyx rename to pyslurm/db/step.pyx index aa1bd612..6e33c8d1 100644 --- a/pyslurm/core/db/step.pyx +++ b/pyslurm/db/step.pyx @@ -24,10 +24,10 @@ from os import WIFSIGNALED, WIFEXITED, WTERMSIG, WEXITSTATUS from pyslurm.core.error import RPCError -from pyslurm.core.db.tres cimport TrackableResources, TrackableResource -from pyslurm.core.common.uint import * -from pyslurm.core.common.ctime import _raw_time -from pyslurm.core.common import ( +from typing import Union +from pyslurm.utils.uint import * +from pyslurm.utils.ctime import _raw_time +from pyslurm.utils.helpers import ( gid_to_name, uid_to_name, instance_to_dict, @@ -53,7 +53,7 @@ cdef class JobStep: cdef JobStep from_ptr(slurmdb_step_rec_t *step): cdef JobStep wrap = JobStep.__new__(JobStep) wrap.ptr = step - wrap.stats = JobStats.from_step(wrap) + wrap.stats = JobStatistics.from_step(wrap) return wrap def as_dict(self): diff --git a/pyslurm/core/db/tres.pxd b/pyslurm/db/tres.pxd similarity index 97% rename from pyslurm/core/db/tres.pxd rename to pyslurm/db/tres.pxd index f08bb3df..40d28799 100644 --- a/pyslurm/core/db/tres.pxd +++ b/pyslurm/db/tres.pxd @@ -21,7 +21,7 @@ # cython: language_level=3 from pyslurm cimport slurm -from pyslurm.core.common cimport cstr +from pyslurm.utils cimport cstr from libc.stdint cimport uint64_t from pyslurm.slurm cimport ( slurmdb_tres_rec_t, diff --git a/pyslurm/core/db/tres.pyx b/pyslurm/db/tres.pyx similarity index 98% rename from pyslurm/core/db/tres.pyx rename to pyslurm/db/tres.pyx index 1e77994b..f4e84130 100644 --- a/pyslurm/core/db/tres.pyx +++ b/pyslurm/db/tres.pyx @@ -22,7 +22,7 @@ # cython: c_string_type=unicode, c_string_encoding=default # cython: language_level=3 -from pyslurm.core.common.uint import * +from pyslurm.utils.uint import * cdef class TrackableResources(dict): diff --git a/pyslurm/core/db/util.pxd b/pyslurm/db/util.pxd similarity index 97% rename from pyslurm/core/db/util.pxd rename to pyslurm/db/util.pxd index deb71ed4..2e9498a6 100644 --- a/pyslurm/core/db/util.pxd +++ b/pyslurm/db/util.pxd @@ -21,7 +21,7 @@ # cython: language_level=3 from pyslurm cimport slurm -from pyslurm.core.common cimport cstr +from pyslurm.utils cimport cstr from pyslurm.slurm cimport ( ListIterator, List, diff --git a/pyslurm/core/db/util.pyx b/pyslurm/db/util.pyx similarity index 100% rename from pyslurm/core/db/util.pyx rename to pyslurm/db/util.pyx diff --git a/pyslurm/utils/__init__.pxd b/pyslurm/utils/__init__.pxd new file mode 100644 index 00000000..7a22bfae --- /dev/null +++ b/pyslurm/utils/__init__.pxd @@ -0,0 +1,2 @@ +# cython: c_string_type=unicode, c_string_encoding=default +# cython: language_level=3 diff --git a/pyslurm/utils/__init__.py b/pyslurm/utils/__init__.py new file mode 100644 index 00000000..eae6e6ed --- /dev/null +++ b/pyslurm/utils/__init__.py @@ -0,0 +1,44 @@ +######################################################################### +# utils.py - pyslurm utility functions +######################################################################### +# Copyright (C) 2023 Toni Harzendorf +# +# This file is part of PySlurm +# +# PySlurm is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. + +# PySlurm is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with PySlurm; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +"""pyslurm utility functions""" + +# Utility time functions +from .ctime import ( + timestr_to_secs, + timestr_to_mins, + secs_to_timestr, + mins_to_timestr, + date_to_timestamp, + timestamp_to_date, +) + +# General utility functions +from .helpers import ( + uid_to_name, + gid_to_name, + user_to_uid, + group_to_gid, + expand_range_str, + humanize, + dehumanize, + nodelist_from_range_str, + nodelist_to_range_str, +) diff --git a/pyslurm/core/common/cstr.pxd b/pyslurm/utils/cstr.pxd similarity index 100% rename from pyslurm/core/common/cstr.pxd rename to pyslurm/utils/cstr.pxd diff --git a/pyslurm/core/common/cstr.pyx b/pyslurm/utils/cstr.pyx similarity index 100% rename from pyslurm/core/common/cstr.pyx rename to pyslurm/utils/cstr.pyx diff --git a/pyslurm/core/common/ctime.pxd b/pyslurm/utils/ctime.pxd similarity index 97% rename from pyslurm/core/common/ctime.pxd rename to pyslurm/utils/ctime.pxd index d8abb12d..b9bde543 100644 --- a/pyslurm/core/common/ctime.pxd +++ b/pyslurm/utils/ctime.pxd @@ -23,7 +23,7 @@ # cython: language_level=3 from pyslurm cimport slurm -from pyslurm.core.common cimport cstr +from pyslurm.utils cimport cstr from libc.stdint cimport uint32_t cdef extern from 'time.h' nogil: diff --git a/pyslurm/core/common/ctime.pyx b/pyslurm/utils/ctime.pyx similarity index 95% rename from pyslurm/core/common/ctime.pyx rename to pyslurm/utils/ctime.pyx index fdf68834..5ffbc424 100644 --- a/pyslurm/core/common/ctime.pyx +++ b/pyslurm/utils/ctime.pyx @@ -33,7 +33,7 @@ def timestr_to_secs(timestr): A Timestring compatible with Slurms time functions. Returns: - int: Amount of time in seconds + (int): Amount of time in seconds """ cdef: char *tmp = NULL @@ -64,7 +64,7 @@ def timestr_to_mins(timestr): A Timestring compatible with Slurms time functions. Returns: - int: Amount of time in minutes + (int): Amount of time in minutes """ cdef: char *tmp = NULL @@ -92,7 +92,7 @@ def secs_to_timestr(secs, default=None): Amount of seconds to convert Returns: - str: A Slurm timestring + (str): A Slurm timestring """ cdef char time_line[32] @@ -122,7 +122,7 @@ def mins_to_timestr(mins, default=None): Amount of minutes to convert Returns: - str: A Slurm timestring + (str): A Slurm timestring """ cdef char time_line[32] @@ -152,7 +152,7 @@ def date_to_timestamp(date, on_nodate=0): A date to convert to a Unix timestamp. Returns: - int: A unix timestamp + (int): A unix timestamp """ cdef: time_t tmp_time @@ -185,7 +185,7 @@ def timestamp_to_date(timestamp): A Unix timestamp that should be converted. Returns: - str: A Slurm date timestring + (str): A Slurm date timestring """ cdef: char time_str[32] diff --git a/pyslurm/core/common/__init__.pxd b/pyslurm/utils/helpers.pxd similarity index 93% rename from pyslurm/core/common/__init__.pxd rename to pyslurm/utils/helpers.pxd index 7915de2f..5de4cf99 100644 --- a/pyslurm/core/common/__init__.pxd +++ b/pyslurm/utils/helpers.pxd @@ -1,5 +1,5 @@ ######################################################################### -# common/__init__.pxd - common/utility functions +# helpers.pxd - basic helper functions ######################################################################### # Copyright (C) 2023 Toni Harzendorf # @@ -25,7 +25,7 @@ from pyslurm cimport slurm from pyslurm.slurm cimport xfree, try_xmalloc, xmalloc from libc.stdint cimport uint8_t, uint16_t, uint32_t, uint64_t -from pyslurm.core.common cimport cstr +from pyslurm.utils cimport cstr from libc.stdlib cimport free cpdef uid_to_name(uint32_t uid, err_on_invalid=*, dict lookup=*) diff --git a/pyslurm/core/common/__init__.pyx b/pyslurm/utils/helpers.pyx similarity index 96% rename from pyslurm/core/common/__init__.pyx rename to pyslurm/utils/helpers.pyx index 6ad5ae47..3617112e 100644 --- a/pyslurm/core/common/__init__.pyx +++ b/pyslurm/utils/helpers.pyx @@ -1,5 +1,5 @@ ######################################################################### -# common/__init__.pyx - common/utility functions +# helpers.pyx - basic helper functions ######################################################################### # Copyright (C) 2023 Toni Harzendorf # @@ -142,7 +142,7 @@ def expand_range_str(range_str): "1,2,3-10,11,15-20" Returns: - list: List of unique values + (list): List of unique values """ ret = [] for mrange in range_str.split(","): @@ -166,7 +166,7 @@ def nodelist_from_range_str(nodelist): and ranges. Returns: - list: List of all nodenames or None on failure + (list): List of all nodenames or None on failure """ if isinstance(nodelist, list): nodelist = ",".join(nodelist) @@ -197,7 +197,7 @@ def nodelist_to_range_str(nodelist): Comma-seperated str or list with unique, unbracketed nodenames. Returns: - str: Bracketed, ranged nodelist or None on failure. + (str): Bracketed, ranged nodelist or None on failure. """ if isinstance(nodelist, list): nodelist = ",".join(nodelist) @@ -233,7 +233,7 @@ def humanize(num, decimals=1): Amount of decimals the humanized string should have. Returns: - str: Humanized number with appropriate suffix. + (str): Humanized number with appropriate suffix. """ if num is None or num == "unlimited": return num @@ -260,7 +260,7 @@ def dehumanize(humanized_str, target="M", decimals=0): Amount of decimal places the result should have. Default is 0 Returns: - int: Dehumanized value + (int): Dehumanized value """ if not humanized_str: return None diff --git a/pyslurm/core/common/uint.pxd b/pyslurm/utils/uint.pxd similarity index 100% rename from pyslurm/core/common/uint.pxd rename to pyslurm/utils/uint.pxd diff --git a/pyslurm/core/common/uint.pyx b/pyslurm/utils/uint.pyx similarity index 100% rename from pyslurm/core/common/uint.pyx rename to pyslurm/utils/uint.pyx diff --git a/tests/integration/test_job_steps.py b/tests/integration/test_job_steps.py index 4ad2de39..bd17a188 100644 --- a/tests/integration/test_job_steps.py +++ b/tests/integration/test_job_steps.py @@ -142,7 +142,7 @@ def test_modify(submit_job): step.modify(JobStep(time_limit="00:05:00")) assert JobStep.load(job, 0).time_limit == 5 - step.modify(time_limit="00:15:00") + step.modify(JobStep(time_limit="00:15:00")) assert JobStep.load(job, 0).time_limit == 15 diff --git a/tests/integration/test_node.py b/tests/integration/test_node.py index 3e1306da..fb6f5197 100644 --- a/tests/integration/test_node.py +++ b/tests/integration/test_node.py @@ -58,7 +58,7 @@ def test_create(): def test_modify(): node = Node(Nodes.load().as_list()[0].name) - node.modify(weight=10000) + node.modify(Node(weight=10000)) assert Node.load(node.name).weight == 10000 node.modify(Node(weight=20000)) diff --git a/tests/unit/test_common.py b/tests/unit/test_common.py index ca3f1cfd..e8bf85a2 100644 --- a/tests/unit/test_common.py +++ b/tests/unit/test_common.py @@ -24,7 +24,7 @@ import pytest import datetime from pyslurm import Job, JobSubmitDescription, Node -from pyslurm.core.common.ctime import ( +from pyslurm.utils.ctime import ( timestr_to_mins, timestr_to_secs, mins_to_timestr, @@ -32,7 +32,7 @@ date_to_timestamp, timestamp_to_date, ) -from pyslurm.core.common.uint import ( +from pyslurm.utils.uint import ( u8, u16, u32, @@ -42,7 +42,7 @@ u32_parse, u64_parse, ) -from pyslurm.core.common import ( +from pyslurm.utils.helpers import ( uid_to_name, gid_to_name, user_to_uid, @@ -56,7 +56,7 @@ nodelist_to_range_str, _sum_prop, ) -from pyslurm.core.common import cstr +from pyslurm.utils import cstr class TestTypes: diff --git a/tests/unit/test_db_slurm_list.py b/tests/unit/test_db_slurm_list.py index 41df371c..6d770bcf 100644 --- a/tests/unit/test_db_slurm_list.py +++ b/tests/unit/test_db_slurm_list.py @@ -22,7 +22,7 @@ import pytest import pyslurm -from pyslurm.core.db.util import SlurmList +from pyslurm.db.util import SlurmList def test_create_and_destroy_list():