Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add support for reverse dependencies of installed modules (WIP) #2783

Open
wants to merge 12 commits into
base: develop
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion easybuild/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@
from easybuild.framework.easyconfig.tweak import obtain_ec_for, tweak
from easybuild.tools.config import find_last_log, get_repository, get_repositorypath, build_option
from easybuild.tools.containers.common import containerize
from easybuild.tools.docs import list_software
from easybuild.tools.docs import list_software, list_reverse_dependencies
from easybuild.tools.filetools import adjust_permissions, cleanup, write_file
from easybuild.tools.github import check_github, find_easybuild_easyconfig, install_github_token
from easybuild.tools.github import close_pr, list_prs, new_pr, merge_pr, update_pr
Expand Down Expand Up @@ -246,6 +246,9 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
detailed = options.list_installed_software == 'detailed'
print(list_software(output_format=options.output_format, detailed=detailed, only_installed=True))

elif options.list_reverse_dependencies:
print(list_reverse_dependencies(output_format=options.output_format))

elif options.list_software:
print(list_software(output_format=options.output_format, detailed=options.list_software == 'detailed'))

Expand All @@ -254,6 +257,7 @@ def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
options.check_github,
options.install_github_token,
options.list_installed_software,
options.list_reverse_dependencies,
options.list_software,
options.close_pr,
options.list_prs,
Expand Down
39 changes: 39 additions & 0 deletions easybuild/tools/docs.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@
from easybuild.tools.filetools import read_file
from easybuild.tools.modules import modules_tool
from easybuild.tools.ordereddict import OrderedDict
from easybuild.tools.run import run_cmd
migueldiascosta marked this conversation as resolved.
Show resolved Hide resolved
from easybuild.tools.toolchain import DUMMY_TOOLCHAIN_NAME
from easybuild.tools.toolchain.utilities import search_toolchain
from easybuild.tools.utilities import import_available_modules, quote_str
Expand Down Expand Up @@ -707,6 +708,44 @@ def list_software_txt(software, detailed=False):
return '\n'.join(lines)


def gather_reverse_dependencies():
"""
Gather reverse dependencies from installed modules.

:return: dictionary with reverse dependencies
"""
reverse_dependencies = {}

module_avail_cmd = "module -t avail 2>&1 | grep -v ':$'"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How would this work with a hierarchical scheme (or any scheme that involved extending the module path)?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That shouldn't be a problem when using modules_tool().available(), which spits out a list of full module names in a hierarchy (i.e. things like Core/GCC/6.4.0 rather than just GCC/6.4.0), so you'll see all modules regardless of where they are in the hierarchy...

module_purge_load_list_cmd = "module purge; module load %s; module -t list 2>&1 | grep -v %s"
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Using purge is troublesome on some systems, and fairly expensive with Lmod...

How about just running this in a subshell, since we only care about the output, not the changes to the environment?

Or use ml show (which only gives direct deps though), that would also make it significantly faster...


available_modules, _ = run_cmd(module_avail_cmd)

for module in available_modules.splitlines():
dependencies, _ = run_cmd(module_purge_load_list_cmd % (module,module))
migueldiascosta marked this conversation as resolved.
Show resolved Hide resolved
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@migueldiascosta probably a good idea to use trace=False here, otherwise you get very amusing output when using eb --trace (which is default for me)

for dependency in dependencies.splitlines():
if dependency not in reverse_dependencies:
reverse_dependencies[dependency] = []
reverse_dependencies[dependency].append(module)

migueldiascosta marked this conversation as resolved.
Show resolved Hide resolved
return reverse_dependencies


def list_reverse_dependencies(output_format=FORMAT_TXT):
"""
Show reverse dependencies of installed software

:param output_format: output format to use
:return: multi-line string presenting requested info
"""

reverse_dependencies = gather_reverse_dependencies()

#TODO: format output
migueldiascosta marked this conversation as resolved.
Show resolved Hide resolved

return reverse_dependencies


def list_toolchains(output_format=FORMAT_TXT):
"""Show list of known toolchains."""
_, all_tcs = search_toolchain('')
Expand Down
2 changes: 2 additions & 0 deletions easybuild/tools/options.py
Original file line number Diff line number Diff line change
Expand Up @@ -558,6 +558,8 @@ def informative_options(self):
'choice', 'store_or_None', 'simple', ['simple', 'detailed']),
'list-installed-software': ("Show list of installed software", 'choice', 'store_or_None', 'simple',
['simple', 'detailed']),
'list-reverse-dependencies': ("Show list of reverse dependencies of installed software", 'choice', 'store_or_None', 'simple',
migueldiascosta marked this conversation as resolved.
Show resolved Hide resolved
['simple', 'detailed']),
migueldiascosta marked this conversation as resolved.
Show resolved Hide resolved
'list-software': ("Show list of supported software", 'choice', 'store_or_None', 'simple',
['simple', 'detailed']),
'list-toolchains': ("Show list of known toolchains",
Expand Down