diff --git a/ci/cli.workflow b/ci/cli.workflow index 89511eb03..2607a536d 100644 --- a/ci/cli.workflow +++ b/ci/cli.workflow @@ -37,12 +37,6 @@ action "test scaffold" { runs = "scaffold" } -action "test recursive" { - needs = "lint" - uses = "./ci/test" - runs = "recursive" -} - action "test dry-run" { needs = "lint" uses = "./ci/test" @@ -127,7 +121,6 @@ action "end" { "test interrupt", "test parallel", "test quiet", - "test recursive", "test reuse", "test scaffold", "test validate", diff --git a/ci/test/ci b/ci/test/ci index 078d9dd34..43bad61b5 100755 --- a/ci/test/ci +++ b/ci/test/ci @@ -93,7 +93,7 @@ EOF git init git add . -git commit -m "popper:run[/tmp/mypaper/myaction/a.workflow --recursive] popper:run[/tmp/mypaper/myaction/b.workflow]" +git commit -m "popper:run[--wfile /tmp/mypaper/myaction/a.workflow] popper:run[--wfile /tmp/mypaper/myaction/b.workflow]" popper run --dry-run --runtime "$RUNTIME" > output grep -q "Found and running workflow at /tmp/mypaper/myaction/a.workflow" output grep -q "Found and running workflow at /tmp/mypaper/myaction/b.workflow" output @@ -113,7 +113,7 @@ action "a" { EOF git add . -git commit -m "Test commit popper:run[/tmp/mypaper/myaction/a.workflow --debug] popper:run[/tmp/mypaper/myaction/d.workflow --quiet]" +git commit -m "Test commit popper:run[--wfile /tmp/mypaper/myaction/a.workflow --debug] popper:run[--wfile /tmp/mypaper/myaction/d.workflow --quiet]" popper run --dry-run --runtime "$RUNTIME" > output grep -q "Found and running workflow at /tmp/mypaper/myaction/a.workflow" output grep -q "Found and running workflow at /tmp/mypaper/myaction/d.workflow" output @@ -140,16 +140,18 @@ action "a2" { EOF git add . -git commit -m "Test commit 2 popper:run[/tmp/mypaper/myaction/e.workflow --skip a1]" +git commit -m "Test commit 2 popper:run[--wfile /tmp/mypaper/myaction/e.workflow --skip a1]" popper run --dry-run --runtime "$RUNTIME" > output grep -q "Found and running workflow at /tmp/mypaper/myaction/e.workflow" output cat output grep -q "a2" output ! grep -q "a1" output -git clone https://github.com/JayjeetAtGithub/test-repo-for-popper.git -cd test-repo-for-popper +git clone https://github.com/JayjeetAtGithub/test-merge-commits.git +cd test-merge-commits popper run --dry-run --runtime "$RUNTIME" > output grep -q "Found and running workflow at b.workflow" output +! grep -q "Found and running workflow at a.workflow" output +! grep -q "Found and running workflow at c.workflow" output echo "Test CI passed." diff --git a/ci/test/dot b/ci/test/dot index 34bc9c0d9..90ce1a9d1 100755 --- a/ci/test/dot +++ b/ci/test/dot @@ -45,7 +45,7 @@ grep "\"Push image to ECR\" -> \"Verify EKS deployment\";" dot_output ## Testing wfile argument feature cd "$test_repo_path" -popper dot example-aws/dummy/main.workflow > dot_output +popper dot --wfile example-aws/dummy/main.workflow > dot_output grep "digraph G .* graph" dot_output grep "\"Verify EKS deployment\" -> \"List Public IP\";" dot_output @@ -65,7 +65,7 @@ grep "\"Push image to ECR\" -> \"Verify EKS deployment\";" dot_output ! grep ".*\"Verify EKS deployment\".*color=" dot_output # test with color -popper dot --colors example-aws/dummy/main.workflow > dot_output +popper dot --colors --wfile example-aws/dummy/main.workflow > dot_output grep ".*\"Verify EKS deployment\".*color=" dot_output echo "Test DOT passed." \ No newline at end of file diff --git a/ci/test/inject b/ci/test/inject index 4ca6e659a..07b1195fc 100755 --- a/ci/test/inject +++ b/ci/test/inject @@ -63,8 +63,8 @@ EOF export POPPER_PRE_WORKFLOW_PATH="$PWD/inject-pre.workflow" export POPPER_POST_WORKFLOW_PATH="$PWD/inject-post.workflow" -popper run --dry-run main.workflow | grep "pre-action" -popper run --dry-run main.workflow | grep "post2-action" +popper run --dry-run | grep "pre-action" +popper run --dry-run | grep "post2-action" echo "Test INJECT ACTIONS passed." \ No newline at end of file diff --git a/ci/test/recursive b/ci/test/recursive deleted file mode 100755 index ac788439e..000000000 --- a/ci/test/recursive +++ /dev/null @@ -1,53 +0,0 @@ -#!/bin/bash -set -ex -# shellcheck source=./ci/test/common -source ./common -init_test_repo -cd "$test_repo_path" - -function create_workflow(){ - cat > "$1" << EOF -workflow "test" { - resolves = "show env" -} - -action "show env" { - uses = "actions/bin/sh@master" - args = "ls" -} -EOF -} - -mkdir another -mkdir another/another - -workflows=( "main.workflow" "another/main.workflow" "another/another/main.workflow" "try.workflow" "another/demo.workflow" ) - -for i in "${workflows[@]}" -do - create_workflow "$i" -done - -popper run --dry-run --recursive --runtime "$RUNTIME" >> out -mapfile -t files < <(grep "^Found .*" out | cut -f6 -d' ' | cut -d'/' -f4-) - -counter=0 - -for workflow in "${workflows[@]}" -do - for file in "${files[@]}" - do - if [ "$workflow" = "$file" ] - then - echo "$file was run!" - echo "success" >> success - fi - done -done - -counter=$(< success wc -l) - -echo "Ran $counter/5 cases successfully" -test "$counter" -eq 5 - -echo "Test RECURSIVE passed." \ No newline at end of file diff --git a/ci/test/samples b/ci/test/samples index 6b0bd55a1..c5dec49c7 100755 --- a/ci/test/samples +++ b/ci/test/samples @@ -7,7 +7,7 @@ cd "$test_repo_path" git clone https://github.com/popperized/spack pushd spack -popper run .ci/test.workflow --runtime "$RUNTIME" +popper run --wfile .ci/test.workflow --runtime "$RUNTIME" popd -echo "Test SAMPLES passed." \ No newline at end of file +echo "Test SAMPLES passed." diff --git a/cli/popper/commands/cmd_dot.py b/cli/popper/commands/cmd_dot.py index 7acf96516..c0a6c11fb 100644 --- a/cli/popper/commands/cmd_dot.py +++ b/cli/popper/commands/cmd_dot.py @@ -5,8 +5,12 @@ from popper.parser import Workflow -@click.argument( - 'wfile', +@click.option( + '--wfile', + help=( + 'File containing the definition of the workflow. ' + '[default: ./github/main.workflow OR ./main.workflow]' + ), required=False, default=None ) diff --git a/cli/popper/commands/cmd_run.py b/cli/popper/commands/cmd_run.py index eb1891ff1..35e6d0e3a 100755 --- a/cli/popper/commands/cmd_run.py +++ b/cli/popper/commands/cmd_run.py @@ -15,9 +15,18 @@ @click.command( 'run', short_help='Run a workflow or action.') @click.argument( - 'target', + 'action', required=False ) +@click.option( + '--wfile', + help=( + 'File containing the definition of the workflow. ' + '[default: ./github/main.workflow OR ./main.workflow]' + ), + required=False, + default=None +) @click.option( '--debug', help=( @@ -53,15 +62,6 @@ required=False, is_flag=True ) -@click.option( - '--recursive', - help=( - 'Run any .workflow file found recursively from current path. ' - 'Ignores flags --on-failure, --skip and --with-dependencies.' - ), - required=False, - is_flag=True -) @click.option( '--reuse', help='Reuse containers between executions (persist container state).', @@ -111,40 +111,30 @@ hidden=True, default=popper.scm.get_git_root_folder() ) - @pass_context def cli(ctx, **kwargs): - """Executes one or more workflows and reports on their status. - - [TARGET] : It can be either path to a workflow file or an action name. - If TARGET is a workflow, the workflow is executed. - If TARGET is an action, the specified action from the default workflow - will be executed. + """Runs a Github Action Workflow. - Examples: + ACTION : The action to execute from a workflow. - 1. When no TARGET argument is passed, Popper will search for the - default workflow (.github/main.workflow or main.workflow) and - execute it if found. + By default, Popper searches for a workflow in .github/main.workflow + or main.workflow and executes it if found. $ popper run - 2. When workflow file is passed as arguments, the specified workflow - will be executed. + When an action name is passed as argument, the specified action + from .github/main.workflow or main.workflow is executed. - $ popper run /path/to/file.workflow + $ popper run myaction - 3. When an action name is passed as argument, Popper will search for - the action in the default workflow and if found, only the action - will be executed. + When an action name is passed as argument and a workflow file + is passed through the `--wfile` option, the specified action from + the specified workflow is executed. - $ popper run myaction + $ popper run --wfile /path/to/main.workflow myaction Note: - * An action argument or options that take action as argument - is not supported in recursive mode. - * When CI is set, popper run searches for special keywords of the form `popper:run[...]`. If found, popper executes with the options given in these run instances else popper executes all the workflows recursively. @@ -156,33 +146,20 @@ def cli(ctx, **kwargs): if popper_run_instances: for args in get_args(popper_run_instances): kwargs.update(args) - if kwargs['recursive']: - log.warn('When CI is set, --recursive is ignored.') - kwargs['recursive'] = False prepare_workflow_execution(**kwargs) else: # If no special keyword is found, we run all the workflows, # recursively. - kwargs['recursive'] = True - prepare_workflow_execution(**kwargs) + prepare_workflow_execution(recursive=True, **kwargs) else: # When CI is not set, prepare_workflow_execution(**kwargs) -def prepare_workflow_execution(**kwargs): +def prepare_workflow_execution(recursive=False, **kwargs): """Set parameters for the workflow execution and run the workflow.""" - def inspect_target(target): - if target: - if target.endswith('.workflow'): - return pu.find_default_wfile(target), None - else: - return pu.find_default_wfile(), target - else: - return pu.find_default_wfile(), target - # Set the logging levels. level = 'ACTION_INFO' if kwargs['quiet']: @@ -199,39 +176,27 @@ def inspect_target(target): kwargs.pop('log_file') # Run the workflow accordingly as recursive/CI and Non-CI. - recursive = kwargs.pop('recursive') - target = kwargs.pop('target') - with_dependencies = kwargs['with_dependencies'] - skip = kwargs['skip'] - on_failure = kwargs['on_failure'] - if recursive: - if target or with_dependencies or skip or on_failure: - # In recursive mode, these flags cannot be used. - log.fail('Any combination of [target] argument, ' - '--with-dependencies , --skip , ' - '--on-failure is invalid in recursive mode.') - for wfile in pu.find_recursive_wfile(): - run_workflow(wfile, target, **kwargs) + kwargs['wfile'] = wfile + run_workflow(**kwargs) else: - wfile, action = inspect_target(target) - run_workflow(wfile, action, **kwargs) + run_workflow(**kwargs) + +def run_workflow(**kwargs): -def run_workflow(wfile, action, **kwargs): - log.info('Found and running workflow at ' + wfile) + kwargs['wfile'] = pu.find_default_wfile(kwargs['wfile']) + log.info('Found and running workflow at ' + kwargs['wfile']) # Initialize a Worklow. During initialization all the validation # takes place automatically. - wf = Workflow(wfile) - - # check for injected actions + wf = Workflow(kwargs['wfile']) + wf_runner = WorkflowRunner(wf) + # Check for injected actions pre_wfile = os.environ.get('POPPER_PRE_WORKFLOW_PATH') post_wfile = os.environ.get('POPPER_POST_WORKFLOW_PATH') - wf_runner = WorkflowRunner(wf) - # Saving workflow instance for signal handling popper.cli.interrupt_params['parallel'] = kwargs['parallel'] @@ -241,39 +206,40 @@ def run_workflow(wfile, action, **kwargs): log.warn("Using --parallel may result in interleaved output. " "You may use --quiet flag to avoid confusion.") - if kwargs['with_dependencies'] and (not action): + if kwargs['with_dependencies'] and (not kwargs['action']): log.fail('`--with-dependencies` can be used only with ' 'action argument.') - if kwargs['skip'] and action: + if kwargs['skip'] and kwargs['action']: log.fail('`--skip` can\'t be used when action argument ' 'is passed.') on_failure = kwargs.pop('on_failure') + wfile = kwargs.pop('wfile') try: if pre_wfile: pre_wf = Workflow(pre_wfile) pre_wf_runner = WorkflowRunner(pre_wf) - pre_wf_runner.run(action, **kwargs) + pre_wf_runner.run(**kwargs) - wf_runner.run(action, **kwargs) + wf_runner.run(**kwargs) if post_wfile: post_wf = Workflow(post_wfile) pre_wf_runner = WorkflowRunner(post_wf) - pre_wf_runner.run(action, **kwargs) + pre_wf_runner.run(**kwargs) except SystemExit as e: if (e.code != 0) and on_failure: kwargs['skip'] = list() - action = on_failure - wf_runner.run(action, **kwargs) + kwargs['action'] = on_failure + wf_runner.run(**kwargs) else: raise - if action: - log.info('Action "{}" finished successfully.'.format(action)) + if kwargs['action']: + log.info('Action "{}" finished successfully.'.format(kwargs['action'])) else: log.info('Workflow "{}" finished successfully.'.format(wfile))