remove compilers from buildtest configuration file

bash_completion.sh

@@ -55,7 +55,7 @@ _avail_executors ()
 # list of available compilers
 _avail_compilers ()
 {
- buildtest config compilers
+ buildtest config compilers list
 }
 
 # list of test ids from report
@@ -247,8 +247,13 @@ _buildtest ()
 
  case "${COMP_WORDS[2+offset]}" in
  compilers|co)
- local opts="--help -h list find test"
- COMPREPLY=( $( compgen -W "${opts}" -- $cur ) )
+ local opts="--help -h"
+ local cmds="list find remove test"
+ local aliases="rm"
+ COMPREPLY=( $( compgen -W "${cmds} ${aliases}" -- $cur ) )
+ if [[ $cur == -* ]] ; then
+ COMPREPLY=( $( compgen -W "$opts" -- $cur ) )
+ fi
 
  if [[ "${prev}" == "list" ]]; then
  local opts="--json --yaml -j -y"
@@ -261,6 +266,9 @@ _buildtest ()
  if [[ "${prev}" == "test" ]]; then
  COMPREPLY=( $( compgen -W "$(_avail_compilers)" -- $cur ) )
  fi
+ if [[ "${prev}" == "remove" ]] || [[ "${prev}" == "rm" ]]; then
+ COMPREPLY=( $( compgen -W "$(_avail_compilers)" -- $cur ) )
+ fi
  ;;
  executors|ex)
  local opts="--help --disabled --invalid --json --yaml -d -h -i -j -y"

buildtest/cli/__init__.py

@@ -1056,6 +1056,12 @@ def config_menu(subparsers, parent_parser):
  metavar="",
  )
  compiler_list = subparsers_compiler.add_parser("list", help="List compilers")
+ compiler_remove = subparsers_compiler.add_parser(
+ "remove", aliases=["rm"], help="Remove compilers"
+ )
+ compiler_remove.add_argument(
+ "compiler_names", nargs="*", help="Specify compiler name to remove"
+ )
  # buildtest config compilers
  compiler_list.add_argument(
  "-j", "--json", action="store_true", help="List compiler details in JSON format"

buildtest/cli/compilers.py

@@ -35,6 +35,49 @@ def compiler_cmd(args, configuration):
  list_compilers(
  configuration=configuration, print_yaml=args.yaml, print_json=args.json
  )
+ if args.compilers in ["remove", "rm"]:
+ remove_compilers(configuration=configuration, names=args.compiler_names)
+
+
+def remove_compilers(configuration, names):
+ """This method will remove compilers from buildtest configuration file and update the configuration file. A list
+ of compiler names are specified, if compiler is found it will be removed.
+
+ Args:
+ configuration (buildtest.config.SiteConfiguration): An instance of SiteConfiguration class
+ names (list): A list of compiler names to remove from configuration file
+
+ """
+ compilers = configuration.target_config["compilers"]["compiler"]
+ updated_compilers = compilers.copy()
+
+ # iterate over the compiler configuration with list of input compiler names to remove. If compiler is found, then delete the key.
+ for compiler_type, compiler_section in compilers.items():
+ for name in names:
+ if name in compiler_section:
+ console.rule(f"Removing compiler name: {name}")
+ console.print(yaml.safe_dump(compiler_section[name]))
+ del compiler_section[name]
+ updated_compilers[compiler_type] = compiler_section
+ # if no compilers are present for a key
+ if len(updated_compilers[compiler_type].keys()) == 0:
+ del updated_compilers[compiler_type]
+
+ configuration.target_config["compilers"]["compiler"] = updated_compilers
+
+ if len(configuration.target_config["compilers"]["compiler"]) == 0:
+ del configuration.target_config["compilers"]["compiler"]
+
+ custom_validator(
+ configuration.config, schema_table["settings.schema.json"]["recipe"]
+ )
+
+ console.print(f"Updating configuration file: {configuration.file}")
+
+ with open(configuration.file, "w") as fd:
+ yaml.safe_dump(
+ configuration.config, fd, default_flow_style=False, sort_keys=False
+ )
 
 
 def list_compilers(configuration, print_yaml=None, print_json=None):
@@ -256,8 +299,9 @@ def __init__(
  self.modulepath = self.modulepath or os.getenv("MODULEPATH")
 
  if not deep_get(self.configuration.target_config, "compilers", "compiler"):
- raise BuildTestError("compiler section not defined")
-
+ raise BuildTestError(
+ "Unable to find any compilers in configuration file. Please define a compiler or detect compilers by running 'buildtest config compilers find'"
+ )
  self.compilers = self.configuration.target_config["compilers"]["compiler"]
 
  self._names = []

buildtest/cli/show.py

@@ -316,6 +316,10 @@ def print_config_show():
  "buildtest config compilers test",
  "Test each compiler instance by performing module load test",
  )
+ table.add_row(
+ "buildtest config compilers remove gcc/7.5.0",
+ "Remove compiler instance gcc/7.5.0",
+ )
  table.add_row(
  "buildtest config profiles list", "Listing all profiles from configuration file"
  )

docs/configuring_buildtest/compilers.rst

@@ -70,7 +70,7 @@ Detect Compilers (Experimental Feature)
 .. Note::
 
  ``buildtest config compilers find`` will not update the buildtest configuration with new compilers, you will need to use ``--update`` option
- to override the configuration file.
+ to write changes back to configuration file.
 
 
 buildtest can detect compilers based on modulefiles and generate compiler section
@@ -86,6 +86,7 @@ In example, below we define a pattern for gcc modules as ``^(gcc)`` which will
 find all modules that start with name `gcc`.
 
 .. code-block:: yaml
+ :emphasize-lines: 2-3
 
  compilers:
  find:
@@ -98,8 +99,8 @@ find all modules that start with name `gcc`.
  fc: /usr/bin/gfortran
 
 
-In this system, we have two gcc modules installed via `spack <https://spack.readthedocs.io/en/latest/>`_
-package manager, we will attempt to add both modules as compiler instance in buildtest.
+In this system, we have two gcc modules and we will add both modules as compiler
+declaration in buildtest.
 
 .. code-block:: console
 
@@ -108,7 +109,7 @@ package manager, we will attempt to add both modules as compiler instance in bui
  gcc/9.3.0-n7p74fd
  gcc/10.2.0-37fmsw7
 
-Next we run ``buildtest config compilers find`` which will search all modules based on
+We will run ``buildtest config compilers find`` which will search all modules based on
 regular expression and add compilers in their respective group. In this example, buildtest
 automatically add ``gcc/9.2.0-n7p74fd`` and ``gcc/10.2.0-37fmsw7`` modules as compiler
 instance. Depending on the compiler group, buildtest will update the properties
@@ -318,7 +319,7 @@ is the programming environment modulefile that will load the GNU compiler on Cra
 
 .. literalinclude:: ../tests/settings/nersc.yml
  :language: yaml
- :emphasize-lines: 6-10
+ :emphasize-lines: 18-22
 
 Now let's run **buildtest config compilers find --detailed** and take note of the generated compilers, you will see that ``PrgEnv-*`` modules will be found in each
 compiler instance under the ``module``, ``load`` section. Furthermore, you will see the cray wrappers **cc**, **CC**, and **ftn** are used
@@ -659,4 +660,99 @@ compiler ``gcc/9.1.01``
  ┃ No. ┃ Compiler Name ┃ Status ┃
  ┡━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━┩
  │ 1 │ gcc/9.1.0 │ ✅ │
- └─────┴───────────────┴────────┘
+ └─────┴───────────────┴────────┘
+
+Removing Compilers
+-------------------
+
+The ``buildtest config compilers remove`` command allows you to remove a compiler from the configuration file.
+This command takes a positional argument that includes the name of compiler you want to remove. buildtest will
+search for compiler name in configuration file and attempt to remove it if it's found.
+
+This command can be used when you find compilers are out of date, for instance you can run ``buildtest config compilers test``
+to test all compilers and show which compilers failed during test. In output below we have
+one failed compiler name ``nvhpc-mixed/23.7``.
+
+.. code-block:: console
+
+ $ buildtest config compilers test
+ Compiler 'builtin_gcc' has no 'modules' defined therefore skipping test
+ Compilers Test Pass
+ ┏━━━━━┳━━━━━━━━━━━━━━━━━━━┳━━━━━━━━┓
+ ┃ No. ┃ Compiler Name ┃ Status ┃
+ ┡━━━━━╇━━━━━━━━━━━━━━━━━━━╇━━━━━━━━┩
+ │ 1 │ builtin_gcc │ ✅ │
+ │ 2 │ gcc/11.2.0 │ ✅ │
+ │ 3 │ gcc/10.3.0 │ ✅ │
+ │ 4 │ gcc/12.2.0 │ ✅ │
+ │ 5 │ gcc/12.1.0 │ ✅ │
+ │ 6 │ cce/15.0.1 │ ✅ │
+ │ 7 │ cce/15.0.0 │ ✅ │
+ │ 8 │ nvhpc/21.11 │ ✅ │
+ │ 9 │ nvhpc/21.9 │ ✅ │
+ │ 10 │ nvhpc/22.7 │ ✅ │
+ │ 11 │ nvhpc-mixed/21.11 │ ✅ │
+ │ 12 │ nvhpc-mixed/21.9 │ ✅ │
+ │ 13 │ nvhpc-mixed/22.7 │ ✅ │
+ │ 14 │ nvhpc/23.1 │ ✅ │
+ │ 15 │ nvhpc-mixed/23.1 │ ✅ │
+ └─────┴───────────────────┴────────┘
+ Compilers Test Fail
+ ┏━━━━━┳━━━━━━━━━━━━━━━━━━┳━━━━━━━━┓
+ ┃ No. ┃ Compiler Name ┃ Status ┃
+ ┡━━━━━╇━━━━━━━━━━━━━━━━━━╇━━━━━━━━┩
+ │ 1 │ nvhpc-mixed/23.7 │ ❌ │
+ └─────┴──────────────────┴────────┘
+
+Note we have tab completion for compilers names that can be removed which can help select the compiler you want
+to remove.
+
+.. code-block::
+
+ $ buildtest config compilers remove
+ builtin_gcc cce/15.0.1 gcc/11.2.0 gcc/12.2.0 nvhpc/21.9 nvhpc/23.1 nvhpc-mixed/21.9 nvhpc-mixed/23.1
+ cce/15.0.0 gcc/10.3.0 gcc/12.1.0 nvhpc/21.11 nvhpc/22.7 nvhpc-mixed/21.11 nvhpc-mixed/22.7 nvhpc-mixed/23.7
+
+Let's remove the compiler `nvhpc-mixed/23.7` by running the following, take note buildtest will update the configuration file
+
+.. code-block::
+
+ $ buildtest config compilers remove nvhpc-mixed/23.7
+ ──────────────────────────────────────────────────────────────────────────────── Removing compiler name: nvhpc-mixed/23.7 ────────────────────────────────────────────────────────────────────────────────
+ cc: cc
+ cxx: CC
+ fc: ftn
+ module:
+ load:
+ - PrgEnv-nvidia
+ - nvhpc-mixed/23.7
+ purge: false
+
+ Updating configuration file: /global/u1/s/siddiq90/gitrepos/buildtest-nersc/config.yml
+
+Now if we rerun the compiler tests, we see all compilers have passed checks
+
+.. code-block:: console
+
+ $ buildtest config compilers test
+ Compiler 'builtin_gcc' has no 'modules' defined therefore skipping test
+ Compilers Test Pass
+ ┏━━━━━┳━━━━━━━━━━━━━━━━━━━┳━━━━━━━━┓
+ ┃ No. ┃ Compiler Name ┃ Status ┃
+ ┡━━━━━╇━━━━━━━━━━━━━━━━━━━╇━━━━━━━━┩
+ │ 1 │ builtin_gcc │ ✅ │
+ │ 2 │ gcc/11.2.0 │ ✅ │
+ │ 3 │ gcc/10.3.0 │ ✅ │
+ │ 4 │ gcc/12.2.0 │ ✅ │
+ │ 5 │ gcc/12.1.0 │ ✅ │
+ │ 6 │ cce/15.0.1 │ ✅ │
+ │ 7 │ cce/15.0.0 │ ✅ │
+ │ 8 │ nvhpc/21.11 │ ✅ │
+ │ 9 │ nvhpc/21.9 │ ✅ │
+ │ 10 │ nvhpc/22.7 │ ✅ │
+ │ 11 │ nvhpc-mixed/21.11 │ ✅ │
+ │ 12 │ nvhpc-mixed/21.9 │ ✅ │
+ │ 13 │ nvhpc-mixed/22.7 │ ✅ │
+ │ 14 │ nvhpc/23.1 │ ✅ │
+ │ 15 │ nvhpc-mixed/23.1 │ ✅ │
+ └─────┴───────────────────┴────────┘

tests/test_nersc.py

@@ -5,7 +5,12 @@
 
 from buildtest.cli.build import BuildTest
 from buildtest.cli.buildspec import BuildspecCache
-from buildtest.cli.compilers import BuildtestCompilers, compiler_find, compiler_test
+from buildtest.cli.compilers import (
+ BuildtestCompilers,
+ compiler_find,
+ compiler_test,
+ remove_compilers,
+)
 from buildtest.config import SiteConfiguration
 from buildtest.defaults import BUILDTEST_ROOT
 from buildtest.system import BuildTestSystem
@@ -107,3 +112,7 @@ def test_compiler_find_alternative_filepath(self):
  temp_path = tempfile.NamedTemporaryFile(dir=os.path.expanduser("~"))
  compiler_find(configuration=self.bc, filepath=temp_path.name)
  temp_path.close()
+
+ def test_compiler_remove(self):
+ compilers = BuildtestCompilers(configuration=self.bc)
+ remove_compilers(configuration=self.bc, names=compilers.names())