From b85861e90ec90978088539dae12c24593cde4ead Mon Sep 17 00:00:00 2001 From: James Gebbie-Rayet Date: Mon, 3 Mar 2025 14:25:43 +0000 Subject: [PATCH 1/6] update for modern workflows --- .github/workflows/ci.yaml | 60 ++++ .travis.yml | 21 -- CHANGELOG.rst | 559 -------------------------------------- MANIFEST.in | 8 - README.rst | 8 +- longbow/__init__.py | 2 + pyproject.toml | 82 ++++++ setup.cfg | 8 - setup.py | 191 ------------- 9 files changed, 146 insertions(+), 793 deletions(-) create mode 100644 .github/workflows/ci.yaml delete mode 100644 .travis.yml delete mode 100644 CHANGELOG.rst delete mode 100644 MANIFEST.in create mode 100644 pyproject.toml delete mode 100644 setup.cfg delete mode 100644 setup.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 0000000..9812111 --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,60 @@ +name: ci + +on: + push: + branches: [master] + pull_request: + +jobs: + + tests: + runs-on: ubuntu-latest + timeout-minutes: 30 + name: Run tests + steps: + - uses: actions/checkout@v4 + + - name: Install Longbow and its testing dependencies + shell: bash + run: pip install -e .[testing] + + - name: Run test suite + shell: bash + run: pytest --cov longbow --cov-report term-missing --cov-append . + + - name: Report Coverage + uses: coverallsapp/github-action@v2 + + docs: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@v4 + - name: Set up Python 3.12 + uses: actions/setup-python@v5 + with: + python-version: 3.12 + - name: Install python dependencies + run: | + pip install --upgrade pip + pip install -e .[docs] + - name: Build docs + run: cd docs && make + + #pre-commit: + # runs-on: ubuntu-latest + # timeout-minutes: 15 + # steps: + # - uses: actions/checkout@v4 + # - name: Set up Python 3.11 + # uses: actions/setup-python@v5 + # with: + # python-version: 3.11 + # - name: Install python dependencies + # run: | + # pip install --upgrade pip + # pip install -e .[pre-commit,docs,testing] + # - name: Run pre-commit + # run: | + # pre-commit install + # pre-commit run --all-files || ( git status --short ; git diff ; exit 1 ) diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 92a3872..0000000 --- a/.travis.yml +++ /dev/null @@ -1,21 +0,0 @@ -language: python -matrix: - include: - - python: 2.7 - - python: 3.5 - - python: 3.6 - - python: 3.7 - dist: xenial - sudo: true -# command to install dependencies -before_install: - - pip install coverage - - pip install coveralls -install: - - pip install . -# command to run tests -script: - - coverage run --source longbow -m py.test - - coverage report -m -after_success: - - coveralls diff --git a/CHANGELOG.rst b/CHANGELOG.rst deleted file mode 100644 index 4b80a12..0000000 --- a/CHANGELOG.rst +++ /dev/null @@ -1,559 +0,0 @@ -Changelog -********* - -Changes to the Longbow source code are listed below by release version. - -Version 1.5.2 -------------- - -1. Bug fix - OMP environment variable added to all schedulers to fix a specific - set of user reported issues (issue #114). - -2. Bug fix - Further PBS and NAMD SMP issues relating to under subscription, - users had to do a hacky way by dropping the corespernode parameter to under - subscribe which resulted in errors from the scheduler. Now users wishing to - do this should set mpiprocs in their job or host conf files to do this (issue #105). - -3. Bug fix - The memory parameter only worked with the PBS scheduler, this has - now been extended to work in all schedulers (issue #98). - -4. Bug fix - The --maxtime parameter went missing from the --help output (issue #96). - -5. Doc fix - The documentation for the recovery mode was incorrect (issue #102). - -6. New feature - An update mode has been added so that users doing - disconnectable sessions can simply run an update to get the current - simulation status and download a snapshot of the data (issue #61). - -7. New feature - Move documentation to be under version control, using sphinx - and readthedocs for auto documentation assembly. Documentation can then - become part of the CI cycle and thus be enforced on code contribution (issue #90). - -8. Enhancement - users can now explicitly set the filenames of stderr and - stdout from the scheduler script using the parameters "stdout = filename" - and "stderr = filename" in their host of job conf files (issue #108). - -9. Enhancement - Users can now make use of existing job scripts, by providing - the name of the script to the parameter "subfile" in their host or job conf - files. This mode is mainly aimed at advanced users that understand the short - falls of doing this and the problems that could occur (issue #77). - -10. Enhancement - Users can now set the naming scheme of the replicate - directories. Instead of having to provide directories of the form - rep1, rep2, ...., repx. Users can now set the name of the "rep" part by - setting the "replicate-naming" parameter. So "replicate-naming = foo" - would need directories named foo1, foo2, ...., foox (issue #92). - -11. Enhancement - Documentation for the examples have been cleaned up and added - to the new sphinx docs (issue #10). - -12. Enhancement - Refactor the exception code in the top level API methods to - remove duplication (issue #44). - -13. Removed support for Python versions 2.6, 3.2 and 3.3 due to these versions - being old unsupported versions and various python packages such as - pip/ci-tools withdrawing support. Longbow may still work for these versions - but this is no longer guaranteed (issue #113). - - -Version 1.5.1 -------------- - -1. Bug fix - a number of parameters that are only used in specific scheduler or - application plugins have been renamed to include the plugin name prefix (issue #54). - -2. Bug fix - executables expressed as absolute paths for supported plugins - would cause a crash due to searching for a module with that key (issue #63). - -3. Bug fix - wrong error message was displayed when an executable didn't exist - on HPC machine (issue #65). - -4. Bug fix - fix for crash when using NAMD SMP builds, the commandline - parameters beginning with "+" would trigger the crash (issue #66). - -5. Bug fix - fixed misleading error messages about missing files and flags (issue #67). - -6. Bug fix - fixed problem where the newly added bash autocomplete did not - allow filenames on disk to autocomplete (issue #68). - -7. Bug fix - fixed a problem when a large number of short jobs that trigger job - subqueuing would cause a crash (issue #72). - -8. Bug fix - fixed user reported bug with strange looking error messages - concealing a further absolute path bug (issue #75). - -9. New feature - Added support for the slurm gres flag so users can do - something like this "slurm-gres = gpu:1" in their host or job conf files (issue #76). - -10. Bug fix - Restored the ability to issue --maxtime on the commandline (issue #78). - -11. New feature - support added for the upcoming release of python chemshell (issue #80). - -12. Bug fix - fix for problem where parameters in configuration files - containing the "=" sign would cause the input file parser to misread - them (issue #81). - -13. Bug fix - fix for problem with LAMMPS jobs where files provided with - "include" or as part of other parameters would not be transferred, thanks - to Anders Johansson for suggesting some ideas and solutions (issue #86). - -14. The structure of the Longbow API has been simplified, the source files no - longer reside in a subdirectory called "core" within the installation - directory. This has made importing much simpler and imports shorter. - -15. Examples have been restructured and the how to run instructions updated. - The actual run files remain the same, there are still incompatibilities - with these and newer versions of MD codes but this will be addressed in - the next version. - - -Version 1.5.0 -------------- - -1. Change of license from GPLv2 to the BSD 3 clause license. This will resolve - the copy-left issues a number of projects are having (Issue #30). - -2. Fix for problem where environment is not properly loaded on some machines, - this is a non-login shell related problem (Issue #4). - -3. Removal of capitalisation from around the whole project for convenience - (Issues #26 #28 #46 #47). - -4. Longbow will now clean out recovery files upon successful completion of the - session (Issue #24). - -5. Bash autocomplete creation added to the setup.py (Issue #23 #49). - -6. Fix for bad exit from recovery mode when user issues a keyboard interrupt - (ctrl-c) (Issue #25). - -7. Fix for monitoring phase causing CPU to be spun up to 100% (Issue #35). - -8. Fix for bug when examples are downloaded in app, the original zip archive - is not removed after extraction (Issue #34). - -9. Fixed a bug in setup script with the python version check code (Issue #32). - -10. Python 3.6 added to supported versions (Issue #33). - -11. All public methods in the Longbow API can now be accessed from a single - import, (ie "import longbow" then "longbow.somemethod()") (Issue #27). - - -Version 1.4.0 -------------- - -This release is a rather large release with a lot of changes to the code and -also to the way things are being done. This project is moving towards using CI -tools more and more, and in this release and all future releases tools for -automated unit testing and code quality checks will be used. - -In this release the following changes have been made in this regard: - -1. Travis-CI is now used to automate testing for all versions of python 2.6 - through 3.5 https://travis-ci.org/HECBioSim/Longbow. - -2. Unit testing added to repository and increased to 100% code coverage. Badges - have also been implemented so users can monitor success rate. Code coverage - is picked up from Travis and compiled at - https://coveralls.io/github/HECBioSim/Longbow. - -3. Automated running of code quality checks is now triggered via github pushes - and is done at https://landscape.io/github/HECBioSim/Longbow - -These tools will eventually be used to test upon merge into the main two -branches of the repo (master and development) and will have tolerances set for -auto-rejection. - -The actual changes to the code are: - -1. Added in some extra commonly found naming for GROMACS executables - (gmx_mpi and gmx_mpi_d). - -2. Bugs detected in shellwrappers.py during unit test writing have now been - fixed. - -3. Fixed formatting issue with logging in staging.py - stage_downstream(). - -4. Removed import statements for relative imports, this eliminated the need for - the try/except imports at the top of each code module. - -5. The parameter "frequency" has been split into two parameters - "polling-frequency" and "staging-frequency". This enables the user to have - Longbow poll jobs without staging all the time. - -6. The timing mechanism for timing between polling events has now been changed - from a disruptive wait() to a timestamp comparison. This stops the process - being "blocked" by wait() and in future will allow the addition of other - features that can happen on different timelines to polling. - -7. There are now two steps to job completion, instead of marking a job as - finished when it is finished on the remote host and results downloaded, - Longbow will now mark at as finished as it is finished on the remote host - and then once staging has happened it will then be marked as complete. - -8. Fixed glitch in applications.py processjobs() where the wrong comparitor was - used on the if statement to construct the upload include list. - -9. Huge refactoring of code to break down larger methods into smaller easier to - test methods. All new methods are private methods (starting with "_") these - should not be used by people making use of Longbow in their own code, unless - they know what they are doing! - -10. Moved all code for Longbow entrypoints out of the executable and into - corelibs/entrypoints.py, this allows more options for integration and also - simplifies it somewhat since the library top level can now be imported from - the library (otherwise hooking against the executable or copying the code - into a project was the only way). - -11. The plugin framework no longer has a complicated path such as - plugins/apps/gromacs.py, now two directories "apps" and "schedulers" sit at - the same level as corelibs in the library. This removed most of the - complexity in the import system, now it is a very simple and elegant way to - provide plugins for these two categories. - -12. Fixed the problem of returning information upon job submission error, this - was a typo in each of the scheduler plugins. - -13. Fix for strange job status glitch when jobid appeard in say a timestamp or - some other parameter in the output of qstat etc, this has been fixed across - all schedulers. - -14. Fixed spacing problem in some generated job submit files. - -15. Modification into the way command-lines are parsed, this is so that the - detection of executables and their commands as well as Longbow commands is - much more robust and can now handle arbitrary executables. - -16. Addition of --nochecks command-line flag, this will disable testing whether - the application is available on the remote host. In some circumstances it - is very difficult to get Longbow to recognise an application is installed. - This is aimed at advanced users only. - -17. All parameters in the main entry point that were previous passed into - longbow have now all been assimilated into one dictionary "parameters" this - then allowed refactoring all the switch cases for command-line parameters - into a much neater single method. - -18. Fix for problem detected during unit test writing for configurations.py - saveconfigs() where if used incorrectly would blow up. - -19. Fixed problem in applications.py causing failed run under python 3.2. - -20. Fix for filenaming glitch when using global files in replicates. - -21. Fixed problem where if required files were not found this was ignored but - should have been flagged up. - -22. Fixed a number of bad initialisers that would cause filenames to go - missing. - -23. legacy code in status method in all schedulers removed. - -24. Fixed freezing glitch when all jobs failed to submit. - -25. Fixed bad parameter in substitutions. - - -Version 1.3.2 -------------- - -1. Fix for new gromacs packaging where the gmx mdrun CLI would not be - recognised. - -2. Fix for case in gromacs where if input file -s and -deffnm would be provided - together and a file was a global file in a replicate job that all files - would be set to output global. - -3. Can now use -deffnm with gromacs on replicate jobs with global files without - having to also set -s. - -4. Fix for missing space in sge replicate script generator. - -5. Added the ability for emails to be sent to the user, these are invoked by - providing the email address in a submit file by - 'email-address = blah@blah.com' and also to set the email flags, these flags - should be the same ones and same format you use in your submit script for - example 'email-flags = -M' or 'email-flags = ib'. - -6. New common executable naming schemes added for things like cuda. - -7. Fixed some missing newline characters from the job submit script generator. - -8. Recovery system broken by renaming the method to same as a variable, this - has now been resolved. - -9. Rewrites to the applications.py module, it was too unwield to add to. This - has now been split down to allow future expansion in a much easier fashion. - - -Version 1.3.1 -------------- - -1. For machines running SGE a new parameter to control the flag used on the -pe - directive has been implemented. This parameter is "sge-peflag" and has a - default value of "mpi", if your cluster requires something different then - use this parameter in a configuration file to set it. - -2. Some cases on SGE clusters, it can be set that even if a job needs only 1 - core that the -pe mpi #cores must be set. A new flag called "sge-peoverride" - will make this happen, just provide "sge-peoverride = true" in a - configuration file to enable. - -3. Fixed a problem where Longbow would exit when no jobs are running, this was - fixed in PBS but not in other schedulers. It is now fixed in all supported - schedulers. - -4. The import statements that looked pretty nasty across the library have been - changed for nicer more pythonic ones. - -5. Some refactoring to get rid of pylint warnings about not using lazy logging. - -6. Ability to add script calls into the job submission script that is created - by Longbow. The parameter "scripts" should be used, for multiple script - references then a comma separated list should be provided. Just add - something like this to your configuration file "scripts = source /some/file" - or "scripts = source /first/file, source /second/file". - -7. Fix a problem when using job configuration files and not referencing a - resource, what should have happened is that Longbow should choose the - default (top one in hosts.conf). But it crashed with a KeyError exception, - this is now fixed. - - -Version 1.3.0 -------------- - -1. Fix for recovery file bug where the file name was mangled by misplaced comma - in os.path.join(). - -2. Fix for bug where the recovery file was not being recognised from the - command-line. - -3. Hydra MPI support was added to the LSF plugin, to get this to work a user - needs to supply mpiexec.hydra as the handler parameter in hosts.conf. - -4. Some bad file keywords have been removed from the NAMD plugin, more files - have been added to the list and some corrections to bad case in list. - -5. Fix for required parameters for staging not being written into the recovery - file. - -6. GROMACS files added to NAMD plugin. - -7. A disconnect feature has been implemented, users supplying --disconnect on - the command-line will have Longbow disconnect after submitting jobs, these - can be reconnected by using the recovery file. - -8. Small change to the executable, all code moved from the if - __name__ == "__main__": to main() and all code previously in main() moved - to longbowmain() so unit tests can be made for this part of the application. - - -Version 1.2.1 -------------- - -1. Fix for annoying error in pip when not using http on urls. - -2. Modified recovery mode to use recovery files that are signed by a time stamp - rather than using a single file, this preserves the ability to have multiple - Longbow instances. - - -Version 1.2.0 -------------- - -1. Simplification of the whole Longbow library. This means lots of changes have - been made to the source code and thus developers making use of Longbow in - their code might be affected, those that simply wrap the executable will be - largely unaffected by this, but may be affected by the below changes. The - main change as part of the API re-write is that the hosts and jobs - structures have simply been merged into a single structure called jobs, the - library is now much more simple to use since all parameters are passed - around in this single structure, so no more checking which data structure a - parameter belongs to. - -2. Extend informative error messages to all schedulers. This is issue is all - about trying to get information from the scheduler as to why a job - submission has failed. - - Error messages from PBS/Torque now passed to the Longbow logging system. - Error messages from LSF now passed to the Longbow logging system. - Error messages from SGE now passed to the Longbow logging system. - Error messages from slurm now passed to the Longbow logging system. - Error messages from SoGE now passed to the Longbow logging system. - Fixed vague error that occurs during staging if bad path is used. - -3. Longbow can now detect queue size limits dynamically. Machines such as - ARCHER that only allow a certain number of jobs in the queued state at any - one time would mean users have to use multiple Longbow sessions to do larger - numbers of jobs. This improvement now means that a single Longbow instance - can now submit a number of jobs larger than this limit and Longbow will hold - back any that cannot get onto the queue in the first submit instance, as - jobs complete, further jobs will be submitted. - -4. A new recovery mode has been introduced that can recover a Longbow session - if for some reason it gets disconnected. See documentation for more details - on this. -5. Removal of the logging module and explicit error handling configuration - within the top level of the library, this issue has ramifications mainly for - developers using Longbow within their applications, users of Longbow won't - be affected much. - - The console output messages have been aligned to 80 character widths for - portability (users might notice this). - Removal of the internal logging module from within Longbow (affects - developers). - - Logging for Longbow is now configured and instantiated at the entry point - of the application (affects developers). - For developers using Longbow as a library now have the ability to specify - how logging happens themselves each Longbow module logs to its own log, - these can then be controlled by the developer allowing for deeper - integration (affects developers). - - The top level longbow method main() has had all unnecessary code and - parameters stripped out, this has all been moved to the application entry - point. This now provides developers with a very high level integration point - for simple integrations (affects developers). - -6. Bad comment on the configuration method fixed (developers only). - -7. API comments have been updated to be more accurate with latest changes. - -8. Fix for unicode/byte string problem between Python2.x and Python3.x which - would garble outputs from subprocess. - -9. Rename the scheduler plugin "sge_arc2" to "soge" (Son of Grid Engine). - -10. All Longbow command-line arguments now support both GNU standard and non - standard forms for convenience (eg. --version and -version would be valid), - although future versions might phase out the non-standard -param in favour - of --param but keeping the single letter -h, -V etc as these are part of - the standard. - - -Version 1.1.6 -------------- - -1. Fix for issue on install through pip where it was complaining about the - missing README.rst. - -2. A manifest file was added to make sure that the setuptools picks up the - extra files. - - -Version 1.1.5 -------------- - -1. Longbow would not install under python 3 using either pip or by running the - setup script manually. This has now been fixed by using print statements of - the format print("text"), this form will work in all versions of python. - -2. Fix for user reported annoyance of installing the hosts.conf in secret, and - also leaving the archive that is downloaded in $home. This has now been - replaced with a warning during install, and also the file is created locally - and not downloaded. - -3. Removed padding from version numbers so v1.01.005 > v1.1.5. - -4. Changed the README.md from markdown format to reStructured text, so now the - README is README.rst, this is to make Longbow more pip friendly whilst still - being bitbucket and possibly github (in case in future we move). - -5. setup.py modified to eliminate the python 3 issues, and also the long - description is now the README.rst, this means that users will see something - standard between the pip page and the bitbucket page, this will also tie in - with new pages on the HECBioSim website which will push pip as the preferred - way to install more prominently. - -6. hosts.conf file has been removed from the development version since this is - now created on install for new users using pip, this will be marked as - deprecated on the website for a while so users on old versions can still get - it. Documentation will need to be updated to reflect this change and provide - the quickstart example that users can grab a copy of if they are doing a more - manual install. However pip will be the encouraged way to install. - -7. Parameters specified within the configuration files had to be specified in a - very strict format (param = value), users that omitted the spaces would find - that Longbow would crash. The code has now been fixed to use regular - expression to read in and parse for the equals sign, this can now handle - cases where users specify cases such as: - - param1 = value1 - param2 =value2 - param3= value3 - param4=value4 - -8. Added python version to logging. - -9. Added longbow version to logging. - -10. Moved version to the top of the longbow file. - -11. Added the paper citation to the logging and readme. - - -Version 1.1.4 -------------- - -1. The saveconfigs method would overwrite entries within the hosts config with - potentially blank entries if it was triggered and parameters existed in such - a state (perhaps via overrides). - -2. There was a missing clause on one of the if statements that prevented - parameters at the hosts level from overriding internal defaults. This - has now been added in. - - -Version 1.1.3 -------------- - -1. Update to the supported command line arguments to support non and GNU - standard options for help and version number. - -2. Update to allow the creation of repX directories when they are missing - (Stops gromacs etc from exiting with path not found error). - -3. Fix for bugs in python 3, there were two issues, one a python name change - and the second was due to garbled output coming from the linux shell - environment. - - -Version 1.1.1 -------------- - -1. Modifications to how Longbow accepts the help and version command line - parameters, some people don't follow GNU standards and thus so that we can - show them the help message/version we have allowed -v, -version, --version, - -h, -help and --help. - -2. Longbow now creates missing repX directories if they are missing in cases - where all input files are global. This would allow jobs that might use - different random seeds on the same input files to be efficient on transfers. - - -Version 1.1.0 -------------- - -1. Fix for overwritten rsync-includes (developers only). - -2. Bad character length on PBS now has an error message to inform the user what - went wrong. Job names longer than 15 chars would be rejected. - -3. Jobs that failed in a multi job would bring down the whole lot due to a bad - initialisation now fixed. - -4. Command line Longbow is submitted with is now logged to the log file for - debugging purposes. - -5. Bug fix with a bad initialisation in job processing code. - -6. New ini parsers that don't rely on python std lib parser, this means - comments are no longer deleted. - -7. The following methods; loadhosts(), loadjobs(), sortjobsconfigs(), - sorthostsconfigs(), amendjobsconfigs() have been replaced with single method - processconfigs(). - -8. Fix for critical failure when the remoteworkdir did not exist. diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 067a7bc..0000000 --- a/MANIFEST.in +++ /dev/null @@ -1,8 +0,0 @@ -# Include AUTHORS file -include AUTHORS - -# Include the license file -include LICENSE - -# Include the readme file -include README.rst \ No newline at end of file diff --git a/README.rst b/README.rst index 5a91ffa..1321b2d 100644 --- a/README.rst +++ b/README.rst @@ -1,15 +1,11 @@ +.. image:: https://travis-ci.org/HECBioSim/Longbow.svg?branch=master + :target: https://travis-ci.org/HECBioSim/Longbow .. image:: https://img.shields.io/pypi/v/Longbow.svg :target: https://pypi.python.org/pypi/Longbow/ -.. image:: https://img.shields.io/pypi/pyversions/Longbow.svg - :target: https://pypi.python.org/pypi/Longbow .. image:: https://img.shields.io/pypi/status/Longbow.svg :target: https://pypi.python.org/pypi/Longbow -.. image:: https://travis-ci.org/HECBioSim/Longbow.svg?branch=master - :target: https://travis-ci.org/HECBioSim/Longbow .. image:: https://coveralls.io/repos/github/HECBioSim/Longbow/badge.svg?branch=master :target: https://coveralls.io/github/HECBioSim/Longbow?branch=master -.. image:: https://api.codacy.com/project/badge/Grade/78370da5b0284083b64e7c1ef030ee8e - :target: https://www.codacy.com/gh/HECBioSim/Longbow?utm_source=github.com&utm_medium=referral&utm_content=HECBioSim/Longbow&utm_campaign=Badge_Grade .. image:: https://readthedocs.org/projects/longbow/badge/?version=latest :target: https://longbow.readthedocs.io/en/latest/?badge=latest :alt: Documentation Status diff --git a/longbow/__init__.py b/longbow/__init__.py index 29f0d9b..914cdc3 100644 --- a/longbow/__init__.py +++ b/longbow/__init__.py @@ -43,3 +43,5 @@ locallist, remotecopy, remotedelete, remotelist, upload, download) from longbow.staging import stage_upstream, stage_downstream, cleanup + +__version__ = "1.5.3" diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..649bcc9 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,82 @@ +[build-system] +# build the package with [flit](https://flit.readthedocs.io) +requires = ["flit_core >=3.4,<4"] +build-backend = "flit_core.buildapi" + +[project] +# See https://www.python.org/dev/peps/pep-0621/ +name = "Longbow" +dynamic = ["version"] # read from longbow/__init__.py +description = "Biomolecular simulation remote job submission tool." +authors = [{name = "James Gebbie-Rayet", email = "james.gebbie@stfc.ac.uk"}] +readme = "README.rst" +license = {file = "LICENSE"} +classifiers = [ + 'Development Status :: 5 - Production/Stable', + 'Environment :: Console', + 'Programming Language :: Python', + 'Topic :: Scientific/Engineering', + 'Topic :: Scientific/Engineering :: Bio-Informatics', + 'Topic :: Scientific/Engineering :: Chemistry', + 'Topic :: System :: Distributed Computing', + 'Topic :: Utilities', + 'Topic :: Software Development :: Libraries', + 'Operating System :: MacOS :: MacOS X', + 'Operating System :: POSIX :: Linux', + 'Operating System :: Unix' +] +keywords = ["aiida", "plugin", "gromacs", "aiida-gromacs"] +requires-python = ">=3.4" +dependencies = [] + +[project.urls] +Source = "https://github.com/HECBioSim/Longbow" + +[project.optional-dependencies] +testing = [ + "pgtest==1.3.2", + "wheel==0.43.0", + "coverage[toml]", + "pytest==8.2.2", + "pytest-cov==5.0.0", + "pytest-sugar==1.0.0" +] +pre-commit = [ + "pre-commit==3.7.1", + "pylint==3.2.5" +] +docs = [ + "sphinx", + "sphinx_rtd_theme", + "sphinxcontrib-contentui", + "sphinxcontrib-details-directive", + "sphinx_copybutton", + "furo", + "markupsafe<2.1" +] + +[project.scripts] +longbow = "longbow.longbow" + +[tool.flit.module] +name = "longbow" + +[tool.pylint.format] +max-line-length = 125 + +[tool.pylint.messages_control] +disable = [ + "too-many-ancestors", + "invalid-name", + "duplicate-code", +] + +[tool.pytest.ini_options] +# Configuration for [pytest](https://docs.pytest.org) +python_files = "test_*.py example_*.py" +filterwarnings = [] + +[tool.coverage.run] +# Configuration of [coverage.py](https://coverage.readthedocs.io) +# reporting which lines of your plugin are covered by tests +source=["longbow"] diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index f609793..0000000 --- a/setup.cfg +++ /dev/null @@ -1,8 +0,0 @@ -[metadata] -description-file = README.rst - -[files] -extra_files = README.rst - -[easy_install] - diff --git a/setup.py b/setup.py deleted file mode 100644 index f4f61f8..0000000 --- a/setup.py +++ /dev/null @@ -1,191 +0,0 @@ -#!/usr/bin/env python - -# BSD 3-Clause License -# -# Copyright (c) 2017, Science and Technology Facilities Council and -# The University of Nottingham -# All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions are met: -# -# * Redistributions of source code must retain the above copyright notice, this -# list of conditions and the following disclaimer. -# -# * Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the documentation -# and/or other materials provided with the distribution. -# -# * Neither the name of the copyright holder nor the names of its -# contributors may be used to endorse or promote products derived from -# this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -# ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE -# LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR -# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF -# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS -# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN -# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) -# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. - -"""Setup script. Used by easy_install and pip.""" - -from distutils.core import setup -import os -import sys - -# Check for unsupported Python versions. -VERSIONS = ['2.7', '3.5', '3.6', '3.7'] -VERSION = str(sys.version_info[0]) + '.' + str(sys.version_info[1]) - -# Do we want config directory with host.conf and bash completion. -HOMECFG = True - -if '--no-home-cfg' in sys.argv: - - HOMECFG = False - sys.argv.pop(sys.argv.index('--no-home-cfg')) - -if VERSION not in VERSIONS: - - sys.exit('The Python version installed is "{0}.{1}", Longbow does not ' - 'support this version. We recommend that you install at least ' - 'version 2.7 or a recent 3.x series'.format(sys.version_info[0], - sys.version_info[1])) - -else: - - print('The Python version installed "{0}.{1}" is supported by Longbow.' - .format(sys.version_info[0], sys.version_info[1])) - -# Setup -setup(name='Longbow', - version='1.5.3-dev', - description='Biomolecular simulation remote job submission tool.', - long_description=open('README.rst').read(), - author='James T Gebbie-Rayet, Gareth B Shannon', - author_email=('james.gebbie@stfc.ac.uk, ' - 'gareth.b.shannon@nasa.gov'), - url='http://www.hecbiosim.ac.uk', - license='OSI Approved :: BSD License', - classifiers=['Development Status :: 5 - Production/Stable', - 'Environment :: Console', - 'Programming Language :: Python', - 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.5', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Topic :: Scientific/Engineering', - 'Topic :: Scientific/Engineering :: Bio-Informatics', - 'Topic :: Scientific/Engineering :: Chemistry', - 'Topic :: System :: Distributed Computing', - 'Topic :: Utilities', - 'Topic :: Software Development :: Libraries', - 'Operating System :: MacOS :: MacOS X', - 'Operating System :: POSIX :: Linux', - 'Operating System :: Unix' - ], - keywords=('hpc hec supercomputer grid cloud batch jobs remote submission' - ' submitter lsf pbs torque sge soge slurm automated staging ' - 'longbow hecbiosim ccpbiosim'), - packages=['longbow', 'longbow.schedulers', 'longbow.apps'], - scripts=['longbow/longbow'], - ) - -if HOMECFG is True: - - # Try and create the .Longbow directory and a basic hosts.conf - try: - - # Setting up the .Longbow directory. - if not os.path.isdir(os.path.expanduser('~/.longbow')): - - print('Longbow will create a hidden directory in your $HOME ' - 'directory in which it will create the hosts configuration ' - 'file. You will need to edit this file with your account ' - 'information on the HPC machines you wish to use. See ' - 'documentation for more information - ' - 'www.hecbiosim.ac.uk/longbow-docs') - - os.mkdir(os.path.expanduser('~/.longbow')) - - HOSTFILE = open(os.path.expanduser('~/.longbow/hosts.conf'), 'w+') - - HOSTFILE.write( - '[QuickStart]\n' - 'host = login.hpc.ac.uk\n' - 'user = myusername\n' - 'corespernode = 24\n' - 'cores = 24\n' - 'remoteworkdir = /work/myusername/\n' - 'account = myaccount\n' - 'modules = mymodules\n') - - HOSTFILE.close() - - else: - - print('Directory already exists at "~/.longbow" - Longbow is ' - 'skipping creating a new one.') - - except IOError: - - print('Longbow failed to create the host configuration file in ' - '"~/.longbow/hosts.conf", you will have to do this manually.\n' - 'The user documentation details the information that should be ' - 'in this file here \n https://longbow.readthedocs.io/en/latest/' - 'usr-getting-started.html#adding-a-hpc-machine-to-longbow') - - # Try to create the bash autocomplete file. - try: - - print('Longbow will try to setup bash autocomplete on this machine, ' - 'this will enable the user to use the tab key as part of the ' - 'longbow command-line to reveal/complete command-line args.\n' - 'This currently only works on some operating systems.') - - BASHFILE = open(os.path.expanduser('~/.longbow/bash_completion'), 'w+') - - BASHFILE.write( - '_longbow()\n' - '{\n' - ' local cur prev opts\n' - ' COMPREPLY=()\n' - ' cur="${COMP_WORDS[COMP_CWORD]}"\n' - ' prev="${COMP_WORDS[COMP_CWORD-1]}"\n' - ' opts="--about --debug --disconnect --examples ' - '--help --hosts --job --jobname --log -- maxtime ' - '--nochecks --recover --resource --replicates ' - '--verbose --version"\n\n' - ' if [[ ${cur} == -* ]]; then\n' - ' COMPREPLY=( $(compgen -W "${opts}" -- ${cur}) )\n' - ' return 0\n' - ' elif [[ ${prev} == --hosts ]] || ' - '[[ ${prev} == --job ]] || [[ ${prev} == --recover ]]; then\n' - ' _filedir\n' - ' fi\n' - '}\n' - 'complete -F _longbow longbow\n') - - BASHFILE.close() - - # Now add a source entry to the user .bashrc - if os.path.isfile(os.path.expanduser('~/.bashrc')): - - BASHFILE = open(os.path.expanduser('~/.bashrc'), 'a+') - - if not any('source ~/.longbow/bash_completion' - in bashline for bashline in BASHFILE.readlines()): - - BASHFILE.write('source ~/.longbow/bash_completion\n') - - except IOError: - - print('Longbow failed to create the bash autocomplete file on this ' - 'machine. Longbow will still continue to function normally, ' - 'however the bash autocomplete will not be available for ' - 'longbow command-line parameters.') From 7a93929295ec0fd30ffa2d59c5213fd003e6c999 Mon Sep 17 00:00:00 2001 From: James Gebbie-Rayet Date: Mon, 3 Mar 2025 14:30:29 +0000 Subject: [PATCH 2/6] fix entrypoint --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 649bcc9..078fbe2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,7 @@ docs = [ ] [project.scripts] -longbow = "longbow.longbow" +longbow = "longbow:launcher" [tool.flit.module] name = "longbow" From 7b1999aafe6672ededc06c52c27b40fa1f0454e9 Mon Sep 17 00:00:00 2001 From: James Gebbie-Rayet Date: Mon, 3 Mar 2025 14:35:25 +0000 Subject: [PATCH 3/6] update badge --- README.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.rst b/README.rst index 1321b2d..1a65101 100644 --- a/README.rst +++ b/README.rst @@ -1,5 +1,5 @@ -.. image:: https://travis-ci.org/HECBioSim/Longbow.svg?branch=master - :target: https://travis-ci.org/HECBioSim/Longbow +.. image:: https://github.com/HECBioSim/Longbow/actions/workflows/ci.yaml/badge.svg + :target: https://github.com/HECBioSim/Longbow/actions/workflows/ci.yaml .. image:: https://img.shields.io/pypi/v/Longbow.svg :target: https://pypi.python.org/pypi/Longbow/ .. image:: https://img.shields.io/pypi/status/Longbow.svg From 37a6ef3cd960ba1118f0c17a8cfeac77a80d989f Mon Sep 17 00:00:00 2001 From: James Gebbie-Rayet Date: Wed, 5 Mar 2025 10:40:57 +0000 Subject: [PATCH 4/6] fix syntax changes --- longbow/applications.py | 4 ++-- longbow/configuration.py | 26 +++++++++++++------------- longbow/entrypoints.py | 6 +++--- longbow/schedulers/lsf.py | 16 ++++++++-------- longbow/schedulers/pbs.py | 14 +++++++------- longbow/schedulers/sge.py | 14 +++++++------- longbow/schedulers/slurm.py | 18 +++++++++--------- longbow/schedulers/soge.py | 14 +++++++------- longbow/scheduling.py | 10 +++++----- longbow/shellwrappers.py | 18 +++++++++--------- 10 files changed, 70 insertions(+), 70 deletions(-) diff --git a/longbow/applications.py b/longbow/applications.py index c89cd77..4bd1d0e 100644 --- a/longbow/applications.py +++ b/longbow/applications.py @@ -100,7 +100,7 @@ def checkapp(jobs): cmd = [] - if jobs[job]["modules"] is "": + if jobs[job]["modules"] == "": LOG.debug("Checking without modules.") @@ -478,7 +478,7 @@ def _procfilesreplicatejobs(app, arg, cwd, initargs, repx): pass # If we have a positive check then file found in rep$i directories. - if tmpitem is not "": + if tmpitem != "": fileitem = tmpitem diff --git a/longbow/configuration.py b/longbow/configuration.py index 248d4c1..c345848 100644 --- a/longbow/configuration.py +++ b/longbow/configuration.py @@ -147,7 +147,7 @@ def processconfigs(parameters): _, hostsections, hostdata = loadconfigs(parameters["hosts"]) # If we have been given a job file then try and load it. - if parameters["job"] is not "": + if parameters["job"] != "": _, _, jobdata = loadconfigs(parameters["job"]) @@ -156,7 +156,7 @@ def processconfigs(parameters): jobdata = {} - if parameters["jobname"] is not "": + if parameters["jobname"] != "": jobname = parameters["jobname"] @@ -277,7 +277,7 @@ def loadconfigs(configfile): params[section] = {} # Find comment markers. - elif len(item) > 0 and item[0] is "#": + elif len(item) > 0 and item[0] == "#": # Ignore comments. pass @@ -294,7 +294,7 @@ def loadconfigs(configfile): params[section][key] = value # Check if there are zero sections. - if len(sections) is 0: + if len(sections) == 0: raise exceptions.ConfigurationError( "Error no sections are defined in configuration file '{0}'" @@ -303,7 +303,7 @@ def loadconfigs(configfile): # Check for sections with zero options. for section in sections: - if len(params[section]) is 0: + if len(params[section]) == 0: raise exceptions.ConfigurationError( "Error section '{0}' contains no parameter definitions using " @@ -438,7 +438,7 @@ def _processconfigsfinalinit(jobs): jobs[job]["jobname"] = job # If the local working directory has not been set, then default to cwd - if jobs[job]["localworkdir"] is "": + if jobs[job]["localworkdir"] == "": jobs[job]["localworkdir"] = os.getcwd() @@ -447,7 +447,7 @@ def _processconfigsfinalinit(jobs): # If modules hasn't been set then try and use a default. If the # executable is given as an absolute path than we will assume the user # knows they need to provide any modules to load etc. - if jobs[job]["modules"] is "": + if jobs[job]["modules"] == "": try: @@ -490,18 +490,18 @@ def _processconfigsparams(jobs, parameters, jobdata, hostdata): if item != "resource": # Command-line overrides are highest priority. - if item in parameters and parameters[item] is not "": + if item in parameters and parameters[item] != "": jobs[job][item] = parameters[item] # Job file is next highest in priority. - elif item in jobdata[job] and jobdata[job][item] is not "": + elif item in jobdata[job] and jobdata[job][item] != "": jobs[job][item] = jobdata[job][item] # Hosts file is next highest in priority. elif item in hostdata[jobs[job]["resource"]] and \ - hostdata[jobs[job]["resource"]][item] is not "": + hostdata[jobs[job]["resource"]][item] != "": jobs[job][item] = hostdata[jobs[job]["resource"]][item] @@ -520,10 +520,10 @@ def _processconfigsresource(parameters, jobdata, hostsections): # Before we go further, check that the job has been assigned a host. try: - if jobdata[job]["resource"] is "": + if jobdata[job]["resource"] == "": # Has a host been named on the command-line? - if parameters["resource"] is not "": + if parameters["resource"] != "": jobs[job]["resource"] = parameters["resource"] @@ -588,7 +588,7 @@ def _processconfigsvalidate(jobs): # Validate required parameters have been set. for validationitem in required: - if jobs[job][validationitem] is "": + if jobs[job][validationitem] == "": raise exceptions.ConfigurationError(required[validationitem]) diff --git a/longbow/entrypoints.py b/longbow/entrypoints.py index 0fe47f4..a6a4f57 100644 --- a/longbow/entrypoints.py +++ b/longbow/entrypoints.py @@ -598,7 +598,7 @@ def _downloadexamples(longbowargs): def _hostfileproc(parameters): """Locate the host configuration file.""" # Hosts - if a filename hasn't been provided default to hosts.conf - if parameters["hosts"] is "": + if parameters["hosts"] == "": parameters["hosts"] = "hosts.conf" @@ -638,7 +638,7 @@ def _jobfileproc(parameters): # Job - if a job configuration file has been supplied but the path hasn't # look in the current working directory and then the execution directory # if needs be. - if parameters["job"] is not "": + if parameters["job"] != "": if os.path.isabs(parameters["job"]) is False: @@ -800,7 +800,7 @@ def _setuplogger(parameters): """ # If no log file name was given then default to "log". - if parameters["log"] is "": + if parameters["log"] == "": parameters["log"] = "longbow.log" diff --git a/longbow/schedulers/lsf.py b/longbow/schedulers/lsf.py index 59e0216..670fe98 100644 --- a/longbow/schedulers/lsf.py +++ b/longbow/schedulers/lsf.py @@ -90,23 +90,23 @@ def prepare(job): jobfile.write("#BSUB -J " + job["jobname"] + "[1-" + job["replicates"] + "]\n") - if job["queue"] is not "": + if job["queue"] != "": jobfile.write("#BSUB -q " + job["queue"] + "\n") - if job["lsf-cluster"] is not "": + if job["lsf-cluster"] != "": jobfile.write("#BSUB -m " + job["lsf-cluster"] + "\n") - if job["memory"] is not "": + if job["memory"] != "": jobfile.write('#BSUB -R "rusage[mem=' + job["memory"] + 'G]"\n') # Account to charge (if supplied). - if job["account"] is not "": + if job["account"] != "": # if no accountflag is provided use the default - if job["accountflag"] is "": + if job["accountflag"] == "": jobfile.write("#BSUB -P " + job["account"] + "\n") @@ -116,9 +116,9 @@ def prepare(job): job["account"] + "\n") # Email user. - if job["email-address"] is not "": + if job["email-address"] != "": - if job["email-flags"] is not "": + if job["email-flags"] != "": jobfile.write("#BSUB " + job["email-flags"] + "\n") @@ -162,7 +162,7 @@ def prepare(job): jobfile.write(item.strip() + "\n") - if job["modules"] is not "": + if job["modules"] != "": for module in job["modules"].split(","): diff --git a/longbow/schedulers/pbs.py b/longbow/schedulers/pbs.py index daaffca..df91c2d 100644 --- a/longbow/schedulers/pbs.py +++ b/longbow/schedulers/pbs.py @@ -90,15 +90,15 @@ def prepare(job): jobfile.write("#PBS -N " + job["jobname"] + "\n") # Queue to submit to (if supplied) - if job["queue"] is not "": + if job["queue"] != "": jobfile.write("#PBS -q " + job["queue"] + "\n") # Account to charge (if supplied). - if job["account"] is not "": + if job["account"] != "": # if no accountflag is provided use the default - if job["accountflag"] is "": + if job["accountflag"] == "": jobfile.write("#PBS -A " + job["account"] + "\n") @@ -128,7 +128,7 @@ def prepare(job): # If user has specified memory append the flag (not all machines support # this). - if job["memory"] is not "": + if job["memory"] != "": tmp = tmp + ":mem=" + job["memory"] + "gb" @@ -136,9 +136,9 @@ def prepare(job): jobfile.write("#PBS -l " + tmp + "\n") # Email user. - if job["email-address"] is not "": + if job["email-address"] != "": - if job["email-flags"] is not "": + if job["email-flags"] != "": jobfile.write("#PBS -m " + job["email-flags"] + "\n") @@ -183,7 +183,7 @@ def prepare(job): jobfile.write("\n") # Load up modules if required. - if job["modules"] is not "": + if job["modules"] != "": for module in job["modules"].split(","): diff --git a/longbow/schedulers/sge.py b/longbow/schedulers/sge.py index 4475c6c..2f47a01 100644 --- a/longbow/schedulers/sge.py +++ b/longbow/schedulers/sge.py @@ -82,15 +82,15 @@ def prepare(job): jobfile.write("#$ -N " + job["jobname"] + "\n") - if job["queue"] is not "": + if job["queue"] != "": jobfile.write("#$ -q " + job["queue"] + "\n") # Account to charge (if supplied). - if job["account"] is not "": + if job["account"] != "": # if no accountflag is provided use the default - if job["accountflag"] is "": + if job["accountflag"] == "": jobfile.write("#$ -A " + job["account"] + "\n") @@ -102,14 +102,14 @@ def prepare(job): jobfile.write("#$ -l h_rt=" + job["maxtime"] + ":00\n") - if job["memory"] is not "": + if job["memory"] != "": jobfile.write("#$ -l h_vmem=" + job["memory"] + "G\n") # Email user. - if job["email-address"] is not "": + if job["email-address"] != "": - if job["email-flags"] is not "": + if job["email-flags"] != "": jobfile.write("#$ -m " + job["email-flags"] + "\n") @@ -153,7 +153,7 @@ def prepare(job): jobfile.write("\n") # Add in module to be loaded. - if job["modules"] is not "": + if job["modules"] != "": for module in job["modules"].split(","): diff --git a/longbow/schedulers/slurm.py b/longbow/schedulers/slurm.py index 345a7e0..83e405b 100644 --- a/longbow/schedulers/slurm.py +++ b/longbow/schedulers/slurm.py @@ -83,15 +83,15 @@ def prepare(job): jobfile.write("#SBATCH -J " + job["jobname"] + "\n") # Queue to submit to (if supplied) - if job["queue"] is not "": + if job["queue"] != "": jobfile.write("#SBATCH -p " + job["queue"] + "\n") # Account to charge (if supplied) - if job["account"] is not "": + if job["account"] != "": # if no accountflag is provided use the default - if job["accountflag"] is "": + if job["accountflag"] == "": jobfile.write("#SBATCH -A " + job["account"] + "\n") @@ -100,19 +100,19 @@ def prepare(job): jobfile.write("#SBATCH " + job["accountflag"] + " " + job["account"] + "\n") - if job["memory"] is not "": + if job["memory"] != "": jobfile.write("#SBATCH --mem=" + job["memory"] + "G" + "\n") # Generic resource (if supplied) - if job["slurm-gres"] is not "": + if job["slurm-gres"] != "": jobfile.write("#SBATCH --gres=" + job["slurm-gres"] + "\n") # Email user. - if job["email-address"] is not "": + if job["email-address"] != "": - if job["email-flags"] is not "": + if job["email-flags"] != "": jobfile.write("#SBATCH --mail-type=" + job["email-flags"] + "\n") @@ -126,7 +126,7 @@ def prepare(job): # If user has specified corespernode for under utilisation then # set the total nodes (-N) parameter. - if cpn is not "": + if cpn != "": nodes = float(cores) / float(cpn) @@ -163,7 +163,7 @@ def prepare(job): jobfile.write("\n") # Load up modules if required. - if job["modules"] is not "": + if job["modules"] != "": for module in job["modules"].split(","): diff --git a/longbow/schedulers/soge.py b/longbow/schedulers/soge.py index 550ece1..e87ce30 100644 --- a/longbow/schedulers/soge.py +++ b/longbow/schedulers/soge.py @@ -83,15 +83,15 @@ def prepare(job): jobfile.write("#$ -N " + job["jobname"] + "\n") - if job["queue"] is not "": + if job["queue"] != "": jobfile.write("#$ -q " + job["queue"] + "\n") # Account to charge (if supplied). - if job["account"] is not "": + if job["account"] != "": # if no accountflag is provided use the default - if job["accountflag"] is "": + if job["accountflag"] == "": jobfile.write("#$ -A " + job["account"] + "\n") @@ -103,14 +103,14 @@ def prepare(job): jobfile.write("#$ -l h_rt=" + job["maxtime"] + ":00\n") - if job["memory"] is not "": + if job["memory"] != "": jobfile.write("#$ -l h_vmem=" + job["memory"] + "G\n") # Email user. - if job["email-address"] is not "": + if job["email-address"] != "": - if job["email-flags"] is not "": + if job["email-flags"] != "": jobfile.write("#$ -m " + job["email-flags"] + "\n") @@ -175,7 +175,7 @@ def prepare(job): jobfile.write("\n") - if job["modules"] is not "": + if job["modules"] != "": for module in job["modules"].split(","): diff --git a/longbow/scheduling.py b/longbow/scheduling.py index 3101652..49f8306 100644 --- a/longbow/scheduling.py +++ b/longbow/scheduling.py @@ -116,7 +116,7 @@ def checkenv(jobs, hostconf): saveparams[job["resource"]] = {} # If we have no scheduler defined by the user then find it. - if job["scheduler"] is "": + if job["scheduler"] == "": _testscheduler(job) saveparams[job["resource"]]["scheduler"] = job["scheduler"] @@ -128,7 +128,7 @@ def checkenv(jobs, hostconf): job["resource"], job["scheduler"]) # If we have no job handler defined by the user then find it. - if job["handler"] is "": + if job["handler"] == "": _testhandler(job) saveparams[job["resource"]]["handler"] = job["handler"] @@ -478,7 +478,7 @@ def _testscheduler(job): LOG.debug("Environment is not '%s'", param) - if job["scheduler"] is "": + if job["scheduler"] == "": raise exceptions.SchedulercheckError("Could not find the job " "scheduling system.") @@ -521,7 +521,7 @@ def _testhandler(job): LOG.debug("The batch queue handler is not '%s'", param) - if job["handler"] is "": + if job["handler"] == "": raise exceptions.HandlercheckError("Could not find the batch queue " "handler.") @@ -553,7 +553,7 @@ def _monitorinitialise(jobs): # If somehow the polling interval parameter is still zero, reduce the # polling to once every 5 minutes. - if pollinterval is 0: + if pollinterval == 0: pollinterval = 300 diff --git a/longbow/shellwrappers.py b/longbow/shellwrappers.py index c11c5b1..2190c47 100644 --- a/longbow/shellwrappers.py +++ b/longbow/shellwrappers.py @@ -252,7 +252,7 @@ def sendtossh(job, args): # This loop is essentially so we can do 3 retries on commands that fail, # this is to catch when things go wrong over SSH like dropped connections, # issues with latency etc. - while i is not 3: + while i != 3: # Send to ssh. shellout = sendtoshell(cmd) @@ -261,11 +261,11 @@ def sendtossh(job, args): # If no error exit loop, if errorcode is not 0 raise exception unless # code is 255 - if errorstate is 0: + if errorstate == 0: break - elif errorstate is 255: + elif errorstate == 255: i = i + 1 @@ -276,7 +276,7 @@ def sendtossh(job, args): "to be sure there are no connection issues.", shellout) # If number of retries hits 3 then give up. - if i is 3: + if i == 3: raise exceptions.SSHError( "SSH failed, make sure a normal terminal can connect to SSH " @@ -325,7 +325,7 @@ def sendtorsync(job, src, dst, includemask, excludemask): port = job["port"] # Figure out if we are using masks to specify files. - if excludemask is not "" and includemask is "": + if excludemask != "" and includemask == "": # Exclude masks are a comma separated list. for mask in excludemask.split(","): @@ -338,7 +338,7 @@ def sendtorsync(job, src, dst, includemask, excludemask): cmd.extend(exclude) cmd.extend(["-e", "ssh -p " + port, src, dst]) - elif excludemask is not "" and includemask is not "": + elif excludemask != "" and includemask != "": # Exclude masks are a comma separated list. for mask in excludemask.split(","): @@ -369,7 +369,7 @@ def sendtorsync(job, src, dst, includemask, excludemask): # This loop is essentially so we can do 3 retries on commands that fail, # this is to catch when things go wrong over SSH like dropped connections, # issues with latency etc. - while i is not 3: + while i != 3: # Send to SSH. shellout = sendtoshell(cmd) @@ -378,7 +378,7 @@ def sendtorsync(job, src, dst, includemask, excludemask): # If no error exit loop, if errorcode is not 0 raise exception unless # code is 255 - if errorstate is 0: + if errorstate == 0: break @@ -387,7 +387,7 @@ def sendtorsync(job, src, dst, includemask, excludemask): i = i + 1 # If number of retries hits 3 then give up. - if i is 3: + if i == 3: raise exceptions.RsyncError( "rsync failed, make sure a normal terminal can connect to " From 5f308ab5338f20f74450f603f6b304ad409f1652 Mon Sep 17 00:00:00 2001 From: James Gebbie-Rayet Date: Wed, 5 Mar 2025 13:54:37 +0000 Subject: [PATCH 5/6] fix failing test - when no longbow config dir exists --- .gitignore | 1 + longbow/scheduling.py | 5 ++--- tests/unit/scheduling/test_monitor.py | 4 ++++ 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 2f49017..2f1b42a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /.project /.pydevproject /.settings/ +*__pycache__* diff --git a/longbow/scheduling.py b/longbow/scheduling.py index 49f8306..7e4a319 100644 --- a/longbow/scheduling.py +++ b/longbow/scheduling.py @@ -248,9 +248,8 @@ def monitor(jobs): saverecoveryfile = _stagejobfiles(jobs, saverecoveryfile) # Save out the recovery files. - if (os.path.isdir(os.path.expanduser('~/.longbow')) and - saverecoveryfile is True and recoveryfileerror is False and - recoveryfile != ""): + if (os.path.isdir(basepath) and saverecoveryfile is True and + recoveryfileerror is False and recoveryfile != ""): saverecoveryfile = False diff --git a/tests/unit/scheduling/test_monitor.py b/tests/unit/scheduling/test_monitor.py index 513647d..49a207c 100644 --- a/tests/unit/scheduling/test_monitor.py +++ b/tests/unit/scheduling/test_monitor.py @@ -195,6 +195,7 @@ def test_monitor_complete1(mock_init, mock_poll, mock_wait, mock_down, assert mock_down.call_count == 1 +@mock.patch('os.path.isdir', mock.MagicMock(return_value="true")) @mock.patch('longbow.configuration.saveini') @mock.patch('longbow.staging.stage_downstream') @mock.patch('longbow.scheduling._checkwaitingjobs') @@ -253,6 +254,7 @@ def test_monitor_complete2(mock_init, mock_poll, mock_wait, mock_down, assert mock_save.call_count == 1 +@mock.patch('os.path.isdir', mock.MagicMock(return_value="true")) @mock.patch('longbow.configuration.saveini') @mock.patch('longbow.staging.stage_downstream') @mock.patch('longbow.scheduling._checkwaitingjobs') @@ -311,6 +313,7 @@ def test_monitor_run1(mock_init, mock_poll, mock_wait, mock_down, assert mock_save.call_count == 1 +@mock.patch('os.path.isdir', mock.MagicMock(return_value="true")) @mock.patch('longbow.configuration.saveini') @mock.patch('longbow.staging.stage_downstream') @mock.patch('longbow.scheduling._checkwaitingjobs') @@ -356,6 +359,7 @@ def test_monitor_except(mock_init, mock_poll, mock_wait, mock_down, assert mock_save.call_count == 1 +@mock.patch('os.path.isdir', mock.MagicMock(return_value="true")) @mock.patch('longbow.configuration.saveini') @mock.patch('longbow.staging.stage_downstream') @mock.patch('longbow.scheduling._checkwaitingjobs') From 4cc98bbb791d9eb65ee7e873055e5aca9fe9191a Mon Sep 17 00:00:00 2001 From: James Gebbie-Rayet Date: Wed, 5 Mar 2025 15:16:41 +0000 Subject: [PATCH 6/6] Create .readthedocs.yml --- .readthedocs.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 .readthedocs.yml diff --git a/.readthedocs.yml b/.readthedocs.yml new file mode 100644 index 0000000..d3093a1 --- /dev/null +++ b/.readthedocs.yml @@ -0,0 +1,17 @@ +version: 2 + +build: + os: "ubuntu-22.04" + tools: + python: "3.11" + +python: + install: + - method: pip + path: . + extra_requirements: + - docs + +sphinx: + # Path to your Sphinx configuration file. + configuration: docs/conf.py