Merge pull request #10 from NREL/pp/rev-updates

reV updates

.github/workflows/pr_rev_cli_tests.yml

@@ -0,0 +1,40 @@
+name: reV CLI Tests
+
+on: pull_request
+
+jobs:
+ build:
+ runs-on: ubuntu-latest
+ steps:
+ - name: checkout gaps
+ uses: actions/checkout@v2
+ with:
+ ref: ${{ github.event.pull_request.head.ref }}
+ fetch-depth: 1
+ path: gaps
+ - name: checkout reV
+ uses: actions/checkout@v2
+ with:
+ repository: nrel/reV
+ fetch-depth: 1
+ path: reV
+ - name: Set up Python
+ uses: conda-incubator/setup-miniconda@v2
+ with:
+ auto-update-conda: true
+ python-version: 3.9
+ - name: Install reV
+ working-directory: ./reV
+ shell: bash -l {0}
+ run: |
+ pip install -e .
+ - name: Install gaps
+ working-directory: ./gaps
+ shell: bash -l {0}
+ run: |
+ pip install -e .[dev]
+ - name: Run reV CLI tests
+ working-directory: ./reV
+ shell: bash -l {0}
+ run: |
+ pytest -k cli -v --disable-warnings

README.rst

@@ -24,21 +24,37 @@ Welcome to Geospatial Analysis Pipelines (GAPs)!
 .. inclusion-intro
 
 Geospatial Analysis Pipelines (GAPs) is a framework designed
-to assist users in scaling their geospatial models to a
-High-Performance Computing (HPC) environment. In particular,
-GAPs automatically distributes the execution of a
-single-location model (such as the `System Advisor Model <https://sam.nrel.gov>`_)
-over a large geospatial extent (e.g. CONUS) across many parallel
-HPC nodes. Born from the open-source `reV <https://github.com/NREL/reV>`_ model, GAPs is a
+to assist researchers and software developers add execution
+tools to their geospatial python models. Born from the
+open-source `reV <https://github.com/NREL/reV>`_ model, GAPs is a
 robust and easy-to-use engine that provides a rich set of features
-such as configuration file generation, job status monitoring,
-CLI Documentation, and more.
-
+such as command-line interface (CLI) generation and documentation,
+basic High-Performance Computing (HPC) scaling capabilities,
+configuration file generation, job status monitoring, and more.
+
+
+Who should use GAPs
+===================
+GAPs is intended to be used by researchers and/or software developers
+who have implemented a working python model but have not yet added any
+external model execution tools. Within minimal effort, developers can
+use GAPs to add a variety of utility for end-users, including a complete
+set of CLI commands and documentation pulled from the model run function
+docstrings. In addition, GAPs provides basic HPC execution capabilities,
+particularly catered towards embarrassingly parallel geospatial models
+(e.g. single-location models such as the `System Advisor Model <https://sam.nrel.gov>`_).
+GAPs can automatically distribute the execution of such models over a large
+geospatial extent (e.g. CONUS) across many parallel HPC nodes.
+
+GAPs is **NOT** a workflow management system (WMS), and therefore does not
+provide any of the in-depth tools/capabilities expected from a proper WMS.
+However, GAPs-supported models can sometimes be included as part of the workflow in
+WMS tools like `Torc <https://pages.github.nrel.gov/viz/wms/index.html#/>`_.
 
 To get started, take a look at the `documentation <https://nrel.github.io/gaps/>`_ (examples coming soon!)
 
 
-Installing gaps
+Installing GAPs
 ===============
 
 NOTE: The installation instruction below assume that you have python installed

gaps/__init__.py

@@ -15,3 +15,4 @@
 logger = logging.getLogger(__name__)
 logger.addHandler(logging.NullHandler())
 logger.setLevel("DEBUG")
+logger.propagate = False

gaps/batch.py

@@ -17,12 +17,7 @@
 
 from rex.utilities import parse_year
 
-from gaps.config import (
- load_config,
- init_logging_from_config,
- ConfigType,
- resolve_all_paths,
-)
+from gaps.config import load_config, ConfigType, resolve_all_paths
 import gaps.cli.pipeline
 from gaps.pipeline import Pipeline
 from gaps.exceptions import gapsValueError, gapsConfigError
@@ -56,8 +51,6 @@ def __init__(self, config):
  self._job_tags = None
  self._base_dir, config = _load_batch_config(config)
  self._pipeline_fp = Path(config["pipeline_config"])
-
- init_logging_from_config(config)
  self._sets = _parse_config(config)
 
  logger.info("Batch job initialized with %d sub jobs.", len(self._sets))
@@ -97,14 +90,12 @@ def _make_job_dirs(self):
 
  # walk through current directory getting everything to copy
  for source_dir, _, filenames in os.walk(self._base_dir):
-
  # don't make additional copies of job sub directories.
  if any(job_tag in source_dir for job_tag in self._sets):
  continue
 
  # For each dir level, iterate through the batch arg combos
  for tag, (arg_comb, mod_files, __) in self._sets.items():
-
  # Add the job tag to the directory path.
  # This will copy config subdirs into the job subdirs
  source_dir = Path(source_dir)

gaps/cli/batch.py

@@ -5,12 +5,15 @@
 import click
 
 import gaps.batch
+from gaps.config import init_logging_from_config_file
 from gaps.cli.command import _WrappedCommand
 from gaps.cli.documentation import _batch_command_help
 
 
 def _batch(config_file, dry, cancel, delete, monitor_background):
  """Execute an analysis pipeline over a parametric set of inputs"""
+ init_logging_from_config_file(config_file, background=monitor_background)
+
  if cancel:
  gaps.batch.BatchJob(config_file).cancel()
  elif delete:
@@ -64,7 +67,11 @@ def batch_command():
  context_settings=None,
  callback=_batch,
  params=params,
- help="Execute an analysis pipeline over a parametric set of inputs",
+ help=(
+ "Execute an analysis pipeline over a parametric set of inputs.\n\n"
+ "The general structure for calling this CLI command is given "
+ "below (add``--help`` to print help info to the terminal)."
+ ),
  epilog=None,
  short_help=None,
  options_metavar="[OPTIONS]",

gaps/cli/cli.py

@@ -16,6 +16,7 @@
  CLICommandFromFunction,
  _WrappedCommand,
 )
+from gaps.cli.documentation import _main_command_help
 from gaps.cli.preprocessing import preprocess_collect_config
 from gaps.cli.status import status_command
 
@@ -56,9 +57,8 @@ def convert_to_commands(self):
  command = as_click_command(command_config)
  self.commands.append(command)
  Pipeline.COMMANDS[command_config.name] = command
- self.template_configs[command_config.name] = (
- command_config.documentation.template_config
- )
+ template_config = command_config.documentation.template_config
+ self.template_configs[command_config.name] = template_config
  return self
 
  def add_pipeline_command(self):
@@ -168,6 +168,8 @@ def make_cli(commands, info=None):
 
  name : str
  Name of program to display at the top of the CLI help.
+ This input is optional, but specifying it yields richer
+ documentation for the main CLI command.
  version : str
  Include program version with a ``--version`` CLI option.
 
@@ -181,6 +183,7 @@ def make_cli(commands, info=None):
  """
  info = info or {}
  command_generator = _CLICommandGenerator(commands)
+ commands = command_generator.generate()
 
  options = [
  click.Option(
@@ -189,12 +192,17 @@ def make_cli(commands, info=None):
  help="Flag to turn on debug logging. Default is not verbose.",
  ),
  ]
- prog_name = [info["name"]] if "name" in info else []
+
+ prog_name = info.get("name")
+ if prog_name:
+ main_help = _main_command_help(
+ prog_name, command_generator.command_configs
+ )
+ else:
+ main_help = "Command Line Interface"
+
  main = click.Group(
- help=" ".join(prog_name + ["Command Line Interface"]),
- params=options,
- callback=_main_cb,
- commands=command_generator.generate(),
+ help=main_help, params=options, callback=_main_cb, commands=commands
  )
  version = info.get("version")
  if version is not None:
@@ -208,3 +216,4 @@ def _main_cb(ctx, verbose):
  """Set the obj and verbose settings of the commands."""
  ctx.ensure_object(dict)
  ctx.obj["VERBOSE"] = verbose
+ ctx.max_content_width = 92