From 910b1e4df08676744e105827de0e25d76202a249 Mon Sep 17 00:00:00 2001 From: tazend Date: Tue, 19 Jul 2022 23:03:27 +0200 Subject: [PATCH 1/3] Rework the build system - Adds a pyproject.toml with build dependencies, so users can also easily do a "pip install" - Update MANIFEST.in file for sdist - add a test_requirements.txt for easy-install of any dependencies necessary for testing - Update README with build instructions Changes in setup.py: - remove Bluegene stuff, it wasn't used anywhere in the code at all - make some cosmetic changes - allow doing things like "sdist" and "clean" without depending on Cython - Depend atleast on Cython 0.29.30 as minimum version now. This way we can mostly make sure that users have recent Cython version to compile Version naming is changed: Make up the pyslurm version from the Slurm Major release (e.g. 22.5) and the current pyslurm patch-level for this major release, so we have for example: 22.5.0 We must make sure (document it) that users don't confuse this with Slurms patch version --- MANIFEST.in | 9 +- README.md | 56 ++-- pyproject.toml | 8 + pyslurm/__version__.py | 2 +- setup.cfg | 2 +- setup.py | 620 +++++++++++++++++++---------------------- test_requirements.txt | 4 + 7 files changed, 325 insertions(+), 376 deletions(-) create mode 100644 pyproject.toml create mode 100644 test_requirements.txt diff --git a/MANIFEST.in b/MANIFEST.in index 57a0ab37..d33c1276 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,11 +1,8 @@ -include pyslurm/alps_cray.h -include pyslurm/bluegene.pxi -include pyslurm/slurm_defines.pxi -include pyslurm/slurm.pxd -include pyslurm/xmalloc.h include README.rst include COPYING.txt -include THANKS.rst graft examples graft tests graft doc +graft pyslurm/slurm +graft pyslurm/pydefines +include pyslurm/alps_cray.h diff --git a/README.md b/README.md index d4530dcc..1bebb43e 100644 --- a/README.md +++ b/README.md @@ -8,56 +8,51 @@ PySlurm is the Python client library for the [Slurm](https://slurm.schedmd.com) ## Prerequisites -* [Slurm](https://slurm.schedmd.com) -* [Python](https://www.python.org) -* [Cython](https://cython.org) +* [Slurm](https://slurm.schedmd.com) - Slurm shared library and header files +* [Python](https://www.python.org) - >= 3.6 +* [Cython](https://cython.org) - >= 0.29.30 but < 3.0 -This PySlurm branch has been tested with: - -* Cython (latest stable) -* Python 3.6, 3.7, 3.8, and 3.9 -* Slurm 22.05 +This PySlurm branch is for the Slurm Major-Release 22.05 ## Installation -You will need to instruct the setup.py script where either the Slurm install -root directory or where the Slurm libraries and Slurm header files are. +By default, it is searched inside `/usr/include` for the Header files and in +`/usr/lib64` for Slurms shared-library (`libslurm.so`) during Installation. +For Slurm installations in different locations, you will need to provide +the corresponding paths to the necessary files. -### Slurm installed using system defaults (/usr) +You can specify these Paths with environment variables, for example: ```shell -python setup.py build -python setup.py install +export SLURM_INCLUDE_DIR=/opt/slurm/22.05/include +export SLURM_LIB_DIR=/opt/slurm/22.05/lib ``` -### Custom installation location +Then you can proceed to install PySlurm, for example: ```shell -python setup.py build --slurm=PATH_TO_SLURM_DIR -python setup.py install +pip install pyslurm==22.05.0 ``` -### Custom Slurm library and include directories +Or by cloning the repository: ```shell -python setup.py build --slurm-lib=PATH_TO_SLURM_LIB --slurm-inc=PATH_TO_SLURM_INC +git clone https://github.com/PySlurm/pyslurm.git && cd pyslurm python setup.py install -``` -### Indicate Blue Gene type Q on build line - -```shell -python setup.py build --bgq +# Or simply with pip +pip install . ``` -### Cleanup build artifacts +Also see `python setup.py --help` -The build will automatically call a cleanup procedure to remove temporary build -files but this can be called directly if needed as well with : +## Release Versioning -```shell -python setup.py clean -``` +PySlurm's versioning scheme follows the official Slurm versioning. The first +two numbers (MAJOR.MINOR) always correspond to Slurms Major-Release, for example +`22.05`. The last number (MICRO) is however not tied in any way to Slurms +MICRO version. For example, any PySlurm 22.05.X version should work with any +Slurm 22.05.X release. ## Documentation @@ -138,8 +133,7 @@ already compiled and installed: ```shell git clone https://github.com/PySlurm/pyslurm.git cd pyslurm -python3.9 setup.py build -python3.9 setup.py install +pip install . ./scripts/configure.sh pipenv sync --dev pipenv run pytest -sv diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..f6dd995d --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,8 @@ +[build-system] +# Minimum requirements +requires = [ + "setuptools==59.2.0", + "wheel==0.37.0", + "Cython>=0.29.30,<3.0", +] + diff --git a/pyslurm/__version__.py b/pyslurm/__version__.py index c7613c95..81a9fc37 100644 --- a/pyslurm/__version__.py +++ b/pyslurm/__version__.py @@ -1 +1 @@ -__version__ = "22.05.2.0" +__version__ = "22.5.0" diff --git a/setup.cfg b/setup.cfg index 0bcd20b0..f23c586d 100644 --- a/setup.cfg +++ b/setup.cfg @@ -16,7 +16,7 @@ build_requires = python-devel >= 2.7 slurm-devel >= 17.11.5 python-nose requires = slurm-slurmd slurm-slurmdbd -use-bzip2 = 1 +use_bzip2 = 1 [build_sphinx] builder = man diff --git a/setup.py b/setup.py index 25db5ca9..602e6bcb 100644 --- a/setup.py +++ b/setup.py @@ -6,24 +6,64 @@ import logging import sys import textwrap -from setuptools import setup +import pathlib +from setuptools import setup, Extension from distutils.dir_util import remove_tree -from distutils.core import Extension from distutils.version import LooseVersion - logger = logging.getLogger(__name__) logging.basicConfig(format="%(levelname)s: %(message)s", level=logging.DEBUG) -CYTHON_VERSION_MIN = "0.19" -SLURM_VERSION = "22.05" - +# Keep in sync with pyproject.toml +CYTHON_VERSION_MIN = "0.29.30" + +SLURM_RELEASE = "22.5" +PYSLURM_PATCH_RELEASE = "0" +SLURM_SHARED_LIB = "libslurmfull.so" +CURRENT_DIR = pathlib.Path(__file__).parent + +metadata = dict( + name="pyslurm", + version=SLURM_RELEASE + "." + PYSLURM_PATCH_RELEASE, + license="GPLv2", + description="Python Interface for Slurm", + long_description=(CURRENT_DIR / "README.md").read_text(), + author="Mark Roberts, Giovanni Torres, et al.", + author_email="pyslurm@googlegroups.com", + url="https://github.com/PySlurm/pyslurm", + platforms=["Linux"], + keywords=["HPC", "Batch Scheduler", "Resource Manager", "Slurm", "Cython"], + packages=["pyslurm"], + classifiers=[ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "License :: OSI Approved :: GNU General Public License v2 (GPLv2)", + "Natural Language :: English", + "Operating System :: POSIX :: Linux", + "Programming Language :: Cython", + "Programming Language :: Python", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Topic :: Software Development :: Libraries", + "Topic :: Software Development :: Libraries :: Python Modules", + "Topic :: System :: Distributed Computing", + ], +) + +class PySlurmConfig(): -def fatal(log_string, code=1): - """Fatal logger""" - logger.error(log_string) - sys.exit(code) + def __init__(self): + # Assume some defaults here + self.slurm_lib = "/usr/lib64" + self.slurm_inc = "/usr/include" + self.slurm_inc_full = "/usr/include/slurm" +config = PySlurmConfig() def warn(log_string): """Warn logger""" @@ -35,350 +75,256 @@ def info(log_string): logger.info(log_string) -try: - from Cython.Distutils import build_ext - from Cython.Compiler.Version import version as cython_version - - if LooseVersion(cython_version) < LooseVersion(CYTHON_VERSION_MIN): - info(f"Cython version %s installed {cython_version}") - fatal(f"Please use Cython version >= {CYTHON_VERSION_MIN}") -except ImportError: - fatal("Cython (https://cython.org) is required to build PySlurm") - fatal(f"Please use Cython version >= {CYTHON_VERSION_MIN}") - - if sys.version_info[:2] < (3, 6): - fatal("Python 3.6 or higher is required to run PySlurm.") + raise RuntimeError("Python 3.6 or higher is required to run PySlurm.") -class Pyslurm: - """Main setup class""" +def usage(): + """Display usage flags""" + print( + textwrap.dedent( + """ + PySlurm Help + ------------ + --slurm-lib=PATH Where to look for libslurm.so (default=/usr/lib64) + You can also instead use the environment + variable SLURM_LIB_DIR. + + --slurm-inc=PATH Where to look for slurm.h, slurm_errno.h + and slurmdb.h (default=/usr/include) + You can also instead use the environment + variable SLURM_INCLUDE_DIR. + + For help with building or installing PySlurm, please ask on the PySlurm + Google group at https://groups.google.com/forum/#!forum/pyslurm. + + If you are sure that you have run into a bug, please report it at + https://github.com/PySlurm/pyslurm/issues. + """ + ) + ) + + +def inc_vers2str(hex_inc_version): + """ + Return a slurm version number string decoded from + the bit shifted components of the slurm version hex + string supplied in slurm.h + """ + a = int(hex_inc_version, 16) + b = (a >> 16 & 0xFF, a >> 8 & 0xFF, a & 0xFF) + # Only really need the major release + return f"{b[0]:02d}.{b[1]:02d}" + + +def read_inc_version(fname): + """ + Read the supplied include file and extract the + slurm version number in the define line e.g + #define SLURM_VERSION_NUMBER 0x020600 + """ + hex_version = "" + with open(fname, "r", encoding="latin-1") as f: + for line in f: + if line.find("#define SLURM_VERSION_NUMBER") == 0: + hex_version = line.split(" ")[2].strip() + info("Detected Slurm version - "f"{inc_vers2str(hex_version)}") + + if not hex_version: + raise RuntimeError("Unable to detect Slurm version") + + return hex_version + + +def find_files_with_extension(path, extensions): + """ + Recursively find all files with specific extensions. + """ + files = [p + for p in pathlib.Path(path).glob("**/*") + if p.suffix in extensions] + + return files + + +def cleanup_build(): + """ + Cleanup build directory and temporary files + """ + info("Checking for objects to clean") + + if os.path.isdir("build"): + info("Removing build/") + remove_tree("build", verbose=1) + + files = find_files_with_extension("pyslurm", {".c", ".pyc"}) + + for file in files: + if file.is_file(): + info(f"Removing: {file}") + file.unlink() + else: + raise RuntimeError(f"{file} is not a file !") + + info("cleanup done") + + +def make_extensions(): + """ + Generate Extension objects from .pyx files + """ + extensions = [] + pyx_files = find_files_with_extension("pyslurm", {".pyx"}) + ext_meta = { + "include_dirs": [config.slurm_inc, "."], + "library_dirs": [config.slurm_lib], + "libraries": ["slurmfull"], + "runtime_library_dirs": [config.slurm_lib], + } + + for pyx in pyx_files: + ext = Extension( + str(pyx.with_suffix("")).replace(os.path.sep, "."), + [str(pyx)], + **ext_meta + ) + extensions.append(ext) + + return extensions + + +def parse_slurm_args(): + args = sys.argv[1:] + + # Check first if necessary paths to Slurm + # header and lib were provided via env var + slurm_lib = os.getenv("SLURM_LIB_DIR") + slurm_inc = os.getenv("SLURM_INCLUDE_DIR") + + # If these are provided, they take precedence + # over the env vars + for arg in args: + if arg.find("--slurm-lib=") == 0: + slurm_lib = arg.split("=")[1] + sys.argv.remove(arg) + if arg.find("--slurm-inc=") == 0: + slurm_inc = arg.split("=")[1] + sys.argv.remove(arg) + + if "--bgq" in args: + config.bgq = 1 + + if slurm_lib: + config.slurm_lib = slurm_lib + if slurm_inc: + config.slurm_inc = slurm_inc + config.slurm_inc_full = os.path.join(slurm_inc, "slurm") + + +def slurm_sanity_checks(): + """ + Check if Slurm headers and Lib exist. + """ + if os.path.exists(f"{config.slurm_lib}/{SLURM_SHARED_LIB}"): + info(f"Found Slurm shared library in {config.slurm_lib}") + else: + raise RuntimeError(f"Cannot locate Slurm shared library in {config.slurm_lib}") + + if os.path.exists(f"{config.slurm_inc_full}/slurm.h"): + info(f"Found Slurm header in {config.slurm_inc_full}") + else: + raise RuntimeError(f"Cannot locate the Slurm include in {config.slurm_inc_full}") + + # Test for Slurm MAJOR.MINOR version match (ignoring .MICRO) + slurm_inc_ver = read_inc_version(f"{config.slurm_inc_full}/slurm_version.h") + + major = (int(slurm_inc_ver, 16) >> 16) & 0xFF + minor = (int(slurm_inc_ver, 16) >> 8) & 0xFF + detected_version = str(major) + "." + str(minor) + + if LooseVersion(detected_version) != LooseVersion(SLURM_RELEASE): + raise RuntimeError( + f"Incorrect slurm version detected, requires Slurm {SLURM_RELEASE}" + ) - def __init__(self): - self.here = os.path.abspath(os.path.dirname(__file__)) - self.about = {} - self.default_slurm_dir = "/usr" - self.slurm_lib = None - self.slurm_inc = None - self.slurm_dir = None - self.bgq = 0 - - # TODO - This needs an recode - file_path = os.path.join(self.here, "pyslurm", "__version__.py") - with open(file_path, "r", encoding="latin-1") as file_object: - for line in file_object: - (key, val) = line.strip("\n").split(' = ') - self.about[key] = val.strip('"') - - @staticmethod - def usage(): - """Display usage flags""" + +def cythongen(): + """ + Build the PySlurm package + """ + info("Building PySlurm from source...") + try: + from Cython.Distutils import build_ext + from Cython.Build import cythonize + from Cython.Compiler.Version import version as cython_version + except ImportError as e: + msg = "Cython (https://cython.org) is required to build PySlurm." + raise RuntimeError(msg) from e + else: + if LooseVersion(cython_version) < LooseVersion(CYTHON_VERSION_MIN): + msg = f"Please use Cython version >= {CYTHON_VERSION_MIN}" + raise RuntimeError(msg) + + + # Clean up temporary build objects first + cleanup_build() + + # Build all extensions + metadata["ext_modules"] = cythonize(make_extensions()) + + +def parse_setuppy_commands(): + """ + Parse the given setup commands + """ + args = sys.argv[1:] + + if not args: + return False + + # Prepend PySlurm help text when passing --help | -h + if "--help" in args or "-h" in args: + usage() print( textwrap.dedent( - """ - PySlurm Help - ------------ - --slurm=PATH Where to look for Slurm, PATH points to - the Slurm installation root (default=/usr) - --slurm-lib=PATH Where to look for libslurm.so (default=/usr/lib64/slurm) - --slurm-inc=PATH Where to look for slurm.h, slurm_errno.h - and slurmdb.h (default=/usr/include) - --bgq Enable support for BG/Q mode - - For help with building or installing PySlurm, please ask on the PySlurm - Google group at https://groups.google.com/forum/#!forum/pyslurm. - - If you are sure that you have run into a bug, please report it at - https://github.com/PySlurm/pyslurm/issues. + """ + Setuptools Help + -------------- """ ) ) + return False - @staticmethod - def scandir(filedir, files=None): - """ - Scan the directory for extension files, converting - them to extension names in dotted notation - """ - if files is None: - files = [] - - for file in os.listdir(filedir): - - path = os.path.join(filedir, file) - - if os.path.isfile(path) and path.endswith(".pyx"): - files.append(path.replace(os.path.sep, ".")[:-4]) - elif os.path.isdir(path): - Pyslurm.scandir(path, files) - - return files - - def make_extension(self, extension_name): - """Generate an Extension object from its dotted name""" - extension_path = extension_name.replace(".", os.path.sep) + ".pyx" - runtime_library_dirs = [self.slurm_lib, f"{self.slurm_lib}/slurm"] - return Extension( - extension_name, - [extension_path], - include_dirs=[self.slurm_inc, "."], - library_dirs=runtime_library_dirs, - libraries=["slurmfull"], - runtime_library_dirs=runtime_library_dirs, - extra_objects=[], - extra_compile_args=[], - extra_link_args=[], - ) - - @staticmethod - def inc_vers2str(hex_inc_version): - """ - Return a slurm version number string decoded from - the bit shifted components of the slurm version hex - string supplied in slurm.h - """ - a = int(hex_inc_version, 16) - b = (a >> 16 & 0xFF, a >> 8 & 0xFF, a & 0xFF) - return f"{b[0]:02d}.{b[1]:02d}.{b[2]:02d}" - - def read_inc_version(self, fname): - """ - Read the supplied include file and extract the - slurm version number in the define line e.g - #define SLURM_VERSION_NUMBER 0x020600 - """ - hex_version = "" - with open(fname, "r", encoding="latin-1") as f: - for line in f: - if line.find("#define SLURM_VERSION_NUMBER") == 0: - hex_version = line.split(" ")[2].strip() - info( - "Build - Detected Slurm version - " - f"{hex_version} {self.inc_vers2str(hex_version)}" - ) - - if not hex_version: - fatal("Build - Unable to detect Slurm version") - - return hex_version - - def check_lib_path(self, slurm_path): - """Check the Slurm library path""" - if not slurm_path: - slurm_path = self.default_slurm_dir - - slurm_path = os.path.normpath(slurm_path) - # if base dir given then search lib64 and then lib - for lib_path in ["lib64", "lib"]: - slurm_lib_path = os.path.join(slurm_path, lib_path) - - if os.path.exists(f"{slurm_lib_path}/libslurm.so"): - info(f"Build - Found Slurm shared library in {slurm_lib_path}") - return slurm_lib_path - - # if base dir given then check this - if os.path.exists(f"{slurm_path}/libslurm.so"): - info(f"Build - Found Slurm shared library in {slurm_path}") - return slurm_path - else: - fatal(f"Build - Cannot locate Slurm shared library in {slurm_path}") - return None - - def create_bluegene_include(self): - """ - Create pyslurm/bluegene.pxi include file. - """ - info("Build - Generating pyslurm/bluegene.pxi file") - try: - with open("pyslurm/bluegene.pxi", "w", encoding="latin-1") as f: - f.write("DEF BG=1\n") - f.write(f"DEF BGQ={self.bgq}\n") - except IOError: - fatal("Build - Unable to write Blue Gene type to pyslurm/bluegene.pxi") - - @staticmethod - def clean(): - """Cleanup build directory and temporary files""" - info("Clean - checking for objects to clean") - - if os.path.isdir("build"): - info("Clean - removing build/") - remove_tree("build", verbose=1) - - files = [ - "pyslurm/__init__.pyc", - "pyslurm/bluegene.pxi", - "pyslurm/pyslurm.c", - "pyslurm/pyslurm.so", - ] - - for file in files: - if os.path.exists(file): - if os.path.isfile(file): - try: - info(f"Clean - removing {file}") - os.unlink(file) - except OSError: - fatal(f"Clean - failed to remove {file}") - else: - fatal(f"Clean - {file} is not a file !") - - info("Clean - completed") - - def build(self): - """Build the PySlurm package""" - info("") - info(f'Building PySlurm ({self.about["__version__"]})') - info("------------------------------") - info("") - info(f"Cython version {cython_version} installed") - info("") - - # Clean up temporary build objects first - self.clean() - - args = sys.argv[1:] - for arg in args: - if arg.find("--slurm=") == 0: - self.slurm_dir = arg.split("=")[1] - sys.argv.remove(arg) - if arg.find("--slurm-lib=") == 0: - self.slurm_lib = arg.split("=")[1] - sys.argv.remove(arg) - if arg.find("--slurm-inc=") == 0: - self.slurm_inc = arg.split("=")[1] - sys.argv.remove(arg) - - # Slurm installation directory - if self.slurm_dir and (self.slurm_lib or self.slurm_inc): - self.usage() - elif self.slurm_dir and not (self.slurm_lib or self.slurm_inc): - self.slurm_lib = self.slurm_dir - self.slurm_inc = f"{self.slurm_dir}/include" - elif not self.slurm_dir and not self.slurm_lib and not self.slurm_inc: - self.slurm_lib = self.default_slurm_dir - self.slurm_inc = f"{self.default_slurm_dir}/include" - elif not self.slurm_dir and (not self.slurm_lib or not self.slurm_inc): - self.usage() - - # Test for slurm.h maybe from derived paths - if os.path.exists(f"{self.slurm_inc}/slurm/slurm.h"): - info(f"Build - Found Slurm header in {self.slurm_inc}/slurm") - elif os.path.exists(f"{self.slurm_inc}/slurm.h"): - info(f"Build - Found Slurm header in {self.slurm_inc}") - else: - fatal(f"Build - Cannot locate the Slurm include in {self.slurm_inc}") + # Clean up all build objects + if "clean" in args: + cleanup_build() + return False - # Test for Slurm MAJOR.MINOR version match (ignoring .MICRO) - try: - slurm_inc_ver = self.read_inc_version(f"{self.slurm_inc}/slurm/slurm_version.h") - except IOError: - slurm_inc_ver = self.read_inc_version(f"{self.slurm_inc}/slurm_version.h") + build_cmd = ('install', 'sdist', 'build', 'build_ext', 'build_py', + 'build_clib', 'build_scripts', 'bdist_wheel', 'bdist_rpm', + 'build_src', 'bdist_egg', 'develop') - major = (int(slurm_inc_ver, 16) >> 16) & 0xFF - minor = (int(slurm_inc_ver, 16) >> 8) & 0xFF + for cmd in build_cmd: + if cmd in args: + return True - if LooseVersion(str(major) + "." + str(minor)) != LooseVersion(SLURM_VERSION): - fatal( - f"Build - Incorrect slurm version detected, requires Slurm {SLURM_VERSION}" - ) + return False - # Test for libslurm in lib64 and then lib - self.slurm_lib = self.check_lib_path(self.slurm_lib) - if not self.slurm_lib: - self.usage() - # BlueGene - self.create_bluegene_include() +def setup_package(): + """ + Define the PySlurm package + """ + build_it = parse_setuppy_commands() - def parse_setuppy_commands(self): - """Parse the given setup commands""" - args = sys.argv[1:] + if build_it: + if "sdist" not in sys.argv: + parse_slurm_args() + slurm_sanity_checks() + cythongen() - if len(args) == 0: - self.usage() - else: - if "--bgq" in args: - self.bgq = 1 - - # Prepend PySlurm help text when passing --help | -h - if "--help" in args or "-h" in args: - self.usage() - print( - textwrap.dedent( - """ - Distutils Help - -------------- - """ - ) - ) - - # Clean up temporary build objects when cleaning - elif "clean" in args: - self.clean() - - # Generate bluegene.pxi when creating source distribution - elif "sdist" in args: - self.create_bluegene_include() - - # --slurm=[ROOT_PATH] - # --slurm-lib=[LIB_PATH] --slurm-inc=[INC_PATH] - elif "build" in args or "build_ext" in args: - self.build() - - def setup_package(self): - """Define the PySlurm package""" - self.parse_setuppy_commands() - - # Get the list of extensions - ext_names = self.scandir("pyslurm/") - - # Build up the set of Extension objects - extensions = [self.make_extension(name) for name in ext_names] - - with open( - os.path.join(self.here, "README.md"), - "r", - encoding="latin-1", - ) as f: - long_description = f.read() - - setup( - name="pyslurm", - version=self.about["__version__"], - license="GPLv2", - description="Python Interface for Slurm", - long_description=long_description, - author="Mark Roberts, Giovanni Torres, et al.", - author_email="pyslurm@googlegroups.com", - url="https://github.com/PySlurm/pyslurm", - platforms=["Linux"], - keywords=["HPC", "Batch Scheduler", "Resource Manager", "Slurm", "Cython"], - packages=["pyslurm"], - install_requires=["Cython"], - ext_modules=extensions, - cmdclass={"build_ext": build_ext}, - classifiers=[ - "Development Status :: 5 - Production/Stable", - "Environment :: Console", - "Intended Audience :: Developers", - "Intended Audience :: System Administrators", - "License :: OSI Approved :: GNU General Public License v2 (GPLv2)", - "Natural Language :: English", - "Operating System :: POSIX :: Linux", - "Programming Language :: Cython", - "Programming Language :: Python", - "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.6", - "Programming Language :: Python :: 3.7", - "Programming Language :: Python :: 3.8", - "Programming Language :: Python :: 3.9", - "Topic :: Software Development :: Libraries", - "Topic :: Software Development :: Libraries :: Python Modules", - "Topic :: System :: Distributed Computing", - ], - ) + setup(**metadata) if __name__ == "__main__": - Pyslurm().setup_package() + setup_package() diff --git a/test_requirements.txt b/test_requirements.txt new file mode 100644 index 00000000..9e2765ff --- /dev/null +++ b/test_requirements.txt @@ -0,0 +1,4 @@ +cython>=0.29.30,<3.0 +wheel==0.37.0 +setuptools==59.2.0 +pytest==7.1.2 From 2afa187179367434ad95a11b8246cc64916109b3 Mon Sep 17 00:00:00 2001 From: tazend Date: Sat, 23 Jul 2022 21:40:23 +0200 Subject: [PATCH 2/3] Disable the auto_pickle feature which was causing that pyslurm wasn't able to be compiled on some kernels. auto pickling may also not be really needed in pyslurm, because by default classes with pointers/structs as attributes aren't generated with pickle support by cython anyway. For more info, check #236 Fixes #236 --- pyslurm/pyslurm.pyx | 1 + 1 file changed, 1 insertion(+) diff --git a/pyslurm/pyslurm.pyx b/pyslurm/pyslurm.pyx index b5605fe0..9e4a0151 100644 --- a/pyslurm/pyslurm.pyx +++ b/pyslurm/pyslurm.pyx @@ -1,6 +1,7 @@ # cython: embedsignature=True # cython: profile=False # cython: language_level=3 +# cython: auto_pickle=False import os import re import sys From b0f5d858544d5eb016ac437f8e6a184b7e28518b Mon Sep 17 00:00:00 2001 From: tazend Date: Sun, 24 Jul 2022 13:16:31 +0200 Subject: [PATCH 3/3] Use libslurm.so instead of libslurmfull.so libslurm should be used for interfacing with the C-API, libslurmfull is more internal to the Slurm tools itself and cannot be guaranteed to be stable when used externally. No functions from libslurmfull were actually used in pyslurm.pyx so we can safely make the switch now. Also removes a few functions in slurm/extra.pxi, which are in libslurmfull but not used anywhere in the code Fixes #209 --- pyslurm/slurm/extra.pxi | 13 ------------- setup.py | 4 ++-- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/pyslurm/slurm/extra.pxi b/pyslurm/slurm/extra.pxi index b4299335..f48572d4 100644 --- a/pyslurm/slurm/extra.pxi +++ b/pyslurm/slurm/extra.pxi @@ -26,12 +26,10 @@ cdef inline void xfree_ptr(void *__p): cdef extern char *slurm_xstrdup(const char *str) - # # Slurm time functions # - cdef extern void slurm_secs2time_str(time_t time, char *string, int size) cdef extern void slurm_mins2time_str(time_t time, char *string, int size) cdef extern int slurm_time_str2mins(const char *string) @@ -43,7 +41,6 @@ cdef extern time_t slurm_parse_time(char *time_str, int past) # Slurm Job functions # - cdef extern void slurm_free_job_desc_msg(job_desc_msg_t *msg) cdef extern void slurm_free_job_info(job_info_t *job) cdef extern void slurm_free_job_info_members(job_info_t *job) @@ -55,16 +52,11 @@ cdef extern char *slurm_job_share_string(uint16_t shared) # # Slurm environment functions -# cdef extern void slurm_env_array_merge(char ***dest_array, const char **src_array) cdef extern char **slurm_env_array_create() cdef extern int slurm_env_array_overwrite(char ***array_ptr, const char *name, const char *value) cdef extern void slurm_env_array_free(char **env_array) -# cdef extern void slurm_env_array_merge_slurm(char ***dest_array, const char **src_array) - - -cdef extern int slurm_select_fini() # # Misc @@ -78,9 +70,4 @@ cdef extern void slurm_free_stats_response_msg (stats_info_response_msg_t *msg) cdef extern int slurm_addto_char_list_with_case(List char_list, char *names, bool lower_case_noralization) cdef extern int slurm_addto_step_list(List step_list, char *names) cdef extern int slurmdb_report_set_start_end_time(time_t *start, time_t *end) -cdef extern int debug_str2flags(char *debug_flags, uint64_t *flags_out) -cdef extern char *debug_flags2str(uint64_t debug_flags) -cdef extern void slurm_sprint_cpu_bind_type(char *str, cpu_bind_type_t cpu_bind_type) cdef extern uint16_t slurm_get_track_wckey() -cdef extern char *select_type_param_string(uint16_t select_type_param) - diff --git a/setup.py b/setup.py index 602e6bcb..4ddce583 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ SLURM_RELEASE = "22.5" PYSLURM_PATCH_RELEASE = "0" -SLURM_SHARED_LIB = "libslurmfull.so" +SLURM_SHARED_LIB = "libslurm.so" CURRENT_DIR = pathlib.Path(__file__).parent metadata = dict( @@ -178,7 +178,7 @@ def make_extensions(): ext_meta = { "include_dirs": [config.slurm_inc, "."], "library_dirs": [config.slurm_lib], - "libraries": ["slurmfull"], + "libraries": ["slurm"], "runtime_library_dirs": [config.slurm_lib], }