Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove recursive and other changes. #711

Merged
merged 4 commits into from
Jul 17, 2019
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 0 additions & 7 deletions ci/cli.workflow
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,6 @@ action "test scaffold" {
runs = "scaffold"
}

action "test recursive" {
needs = "lint"
uses = "./ci/test"
runs = "recursive"
}

action "test dry-run" {
needs = "lint"
uses = "./ci/test"
Expand Down Expand Up @@ -127,7 +121,6 @@ action "end" {
"test interrupt",
"test parallel",
"test quiet",
"test recursive",
"test reuse",
"test scaffold",
"test validate",
Expand Down
12 changes: 7 additions & 5 deletions ci/test/ci
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ EOF

git init
git add .
git commit -m "popper:run[/tmp/mypaper/myaction/a.workflow --recursive] popper:run[/tmp/mypaper/myaction/b.workflow]"
git commit -m "popper:run[--wfile /tmp/mypaper/myaction/a.workflow] popper:run[--wfile /tmp/mypaper/myaction/b.workflow]"
popper run --dry-run --runtime "$RUNTIME" > output
grep -q "Found and running workflow at /tmp/mypaper/myaction/a.workflow" output
grep -q "Found and running workflow at /tmp/mypaper/myaction/b.workflow" output
Expand All @@ -113,7 +113,7 @@ action "a" {
EOF

git add .
git commit -m "Test commit popper:run[/tmp/mypaper/myaction/a.workflow --debug] popper:run[/tmp/mypaper/myaction/d.workflow --quiet]"
git commit -m "Test commit popper:run[--wfile /tmp/mypaper/myaction/a.workflow --debug] popper:run[--wfile /tmp/mypaper/myaction/d.workflow --quiet]"
popper run --dry-run --runtime "$RUNTIME" > output
grep -q "Found and running workflow at /tmp/mypaper/myaction/a.workflow" output
grep -q "Found and running workflow at /tmp/mypaper/myaction/d.workflow" output
Expand All @@ -140,16 +140,18 @@ action "a2" {
EOF

git add .
git commit -m "Test commit 2 popper:run[/tmp/mypaper/myaction/e.workflow --skip a1]"
git commit -m "Test commit 2 popper:run[--wfile /tmp/mypaper/myaction/e.workflow --skip a1]"
popper run --dry-run --runtime "$RUNTIME" > output
grep -q "Found and running workflow at /tmp/mypaper/myaction/e.workflow" output
cat output
grep -q "a2" output
! grep -q "a1" output

git clone https://github.com/JayjeetAtGithub/test-repo-for-popper.git
cd test-repo-for-popper
git clone https://github.com/JayjeetAtGithub/test-merge-commits.git
cd test-merge-commits
popper run --dry-run --runtime "$RUNTIME" > output
grep -q "Found and running workflow at b.workflow" output
! grep -q "Found and running workflow at a.workflow" output
! grep -q "Found and running workflow at c.workflow" output

echo "Test CI passed."
4 changes: 2 additions & 2 deletions ci/test/dot
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ grep "\"Push image to ECR\" -> \"Verify EKS deployment\";" dot_output

## Testing wfile argument feature
cd "$test_repo_path"
popper dot example-aws/dummy/main.workflow > dot_output
popper dot --wfile example-aws/dummy/main.workflow > dot_output

grep "digraph G .* graph" dot_output
grep "\"Verify EKS deployment\" -> \"List Public IP\";" dot_output
Expand All @@ -65,7 +65,7 @@ grep "\"Push image to ECR\" -> \"Verify EKS deployment\";" dot_output
! grep ".*\"Verify EKS deployment\".*color=" dot_output

# test with color
popper dot --colors example-aws/dummy/main.workflow > dot_output
popper dot --colors --wfile example-aws/dummy/main.workflow > dot_output
grep ".*\"Verify EKS deployment\".*color=" dot_output

echo "Test DOT passed."
4 changes: 2 additions & 2 deletions ci/test/inject
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,8 @@ EOF
export POPPER_PRE_WORKFLOW_PATH="$PWD/inject-pre.workflow"
export POPPER_POST_WORKFLOW_PATH="$PWD/inject-post.workflow"

popper run --dry-run main.workflow | grep "pre-action"
popper run --dry-run main.workflow | grep "post2-action"
popper run --dry-run | grep "pre-action"
popper run --dry-run | grep "post2-action"


echo "Test INJECT ACTIONS passed."
53 changes: 0 additions & 53 deletions ci/test/recursive

This file was deleted.

4 changes: 2 additions & 2 deletions ci/test/samples
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ cd "$test_repo_path"

git clone https://github.com/popperized/spack
pushd spack
popper run .ci/test.workflow --runtime "$RUNTIME"
popper run --wfile .ci/test.workflow --runtime "$RUNTIME"
popd

echo "Test SAMPLES passed."
echo "Test SAMPLES passed."
8 changes: 6 additions & 2 deletions cli/popper/commands/cmd_dot.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,12 @@
from popper.parser import Workflow


@click.argument(
'wfile',
@click.option(
'--wfile',
help=(
'File containing the definition of the workflow. '
'[default: ./github/main.workflow OR ./main.workflow]'
),
required=False,
default=None
)
Expand Down
120 changes: 43 additions & 77 deletions cli/popper/commands/cmd_run.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,18 @@
@click.command(
'run', short_help='Run a workflow or action.')
@click.argument(
'target',
'action',
required=False
)
@click.option(
'--wfile',
help=(
'File containing the definition of the workflow. '
'[default: ./github/main.workflow OR ./main.workflow]'
),
required=False,
default=None
)
@click.option(
'--debug',
help=(
Expand Down Expand Up @@ -53,15 +62,6 @@
required=False,
is_flag=True
)
@click.option(
'--recursive',
help=(
'Run any .workflow file found recursively from current path. '
'Ignores flags --on-failure, --skip and --with-dependencies.'
),
required=False,
is_flag=True
)
@click.option(
'--reuse',
help='Reuse containers between executions (persist container state).',
Expand Down Expand Up @@ -111,40 +111,30 @@
hidden=True,
default=popper.scm.get_git_root_folder()
)

@pass_context
def cli(ctx, **kwargs):
"""Executes one or more workflows and reports on their status.

[TARGET] : It can be either path to a workflow file or an action name.
If TARGET is a workflow, the workflow is executed.
If TARGET is an action, the specified action from the default workflow
will be executed.
"""Runs a Github Action Workflow.

Examples:
ACTION : The action to execute from a workflow.

1. When no TARGET argument is passed, Popper will search for the
default workflow (.github/main.workflow or main.workflow) and
execute it if found.
By default, Popper searches for a workflow in .github/main.workflow
or main.workflow and executes it if found.

$ popper run

2. When workflow file is passed as arguments, the specified workflow
will be executed.
When an action name is passed as argument, the specified action
from .github/main.workflow or main.workflow is executed.

$ popper run /path/to/file.workflow
$ popper run myaction

3. When an action name is passed as argument, Popper will search for
the action in the default workflow and if found, only the action
will be executed.
When an action name is passed as argument and a workflow file
is passed through the `--wfile` option, the specified action from
the specified workflow is executed.

$ popper run myaction
$ popper run --wfile /path/to/main.workflow myaction

Note:

* An action argument or options that take action as argument
is not supported in recursive mode.

* When CI is set, popper run searches for special keywords of the form
`popper:run[...]`. If found, popper executes with the options given in
these run instances else popper executes all the workflows recursively.
Expand All @@ -156,33 +146,20 @@ def cli(ctx, **kwargs):
if popper_run_instances:
for args in get_args(popper_run_instances):
kwargs.update(args)
if kwargs['recursive']:
log.warn('When CI is set, --recursive is ignored.')
kwargs['recursive'] = False
prepare_workflow_execution(**kwargs)
else:
# If no special keyword is found, we run all the workflows,
# recursively.
kwargs['recursive'] = True
prepare_workflow_execution(**kwargs)
prepare_workflow_execution(recursive=True, **kwargs)
else:
# When CI is not set,
prepare_workflow_execution(**kwargs)


def prepare_workflow_execution(**kwargs):
def prepare_workflow_execution(recursive=False, **kwargs):
"""Set parameters for the workflow execution
and run the workflow."""

def inspect_target(target):
if target:
if target.endswith('.workflow'):
return pu.find_default_wfile(target), None
else:
return pu.find_default_wfile(), target
else:
return pu.find_default_wfile(), target

# Set the logging levels.
level = 'ACTION_INFO'
if kwargs['quiet']:
Expand All @@ -199,39 +176,27 @@ def inspect_target(target):
kwargs.pop('log_file')

# Run the workflow accordingly as recursive/CI and Non-CI.
recursive = kwargs.pop('recursive')
target = kwargs.pop('target')
with_dependencies = kwargs['with_dependencies']
skip = kwargs['skip']
on_failure = kwargs['on_failure']

if recursive:
if target or with_dependencies or skip or on_failure:
# In recursive mode, these flags cannot be used.
log.fail('Any combination of [target] argument, '
'--with-dependencies <action>, --skip <action>, '
'--on-failure <action> is invalid in recursive mode.')

for wfile in pu.find_recursive_wfile():
run_workflow(wfile, target, **kwargs)
kwargs['wfile'] = wfile
run_workflow(**kwargs)
else:
wfile, action = inspect_target(target)
run_workflow(wfile, action, **kwargs)
run_workflow(**kwargs)


def run_workflow(**kwargs):

def run_workflow(wfile, action, **kwargs):
log.info('Found and running workflow at ' + wfile)
kwargs['wfile'] = pu.find_default_wfile(kwargs['wfile'])
log.info('Found and running workflow at ' + kwargs['wfile'])
# Initialize a Worklow. During initialization all the validation
# takes place automatically.
wf = Workflow(wfile)

# check for injected actions
wf = Workflow(kwargs['wfile'])
wf_runner = WorkflowRunner(wf)

# Check for injected actions
pre_wfile = os.environ.get('POPPER_PRE_WORKFLOW_PATH')
post_wfile = os.environ.get('POPPER_POST_WORKFLOW_PATH')

wf_runner = WorkflowRunner(wf)

# Saving workflow instance for signal handling
popper.cli.interrupt_params['parallel'] = kwargs['parallel']

Expand All @@ -241,39 +206,40 @@ def run_workflow(wfile, action, **kwargs):
log.warn("Using --parallel may result in interleaved output. "
"You may use --quiet flag to avoid confusion.")

if kwargs['with_dependencies'] and (not action):
if kwargs['with_dependencies'] and (not kwargs['action']):
log.fail('`--with-dependencies` can be used only with '
'action argument.')

if kwargs['skip'] and action:
if kwargs['skip'] and kwargs['action']:
log.fail('`--skip` can\'t be used when action argument '
'is passed.')

on_failure = kwargs.pop('on_failure')
wfile = kwargs.pop('wfile')

try:
if pre_wfile:
pre_wf = Workflow(pre_wfile)
pre_wf_runner = WorkflowRunner(pre_wf)
pre_wf_runner.run(action, **kwargs)
pre_wf_runner.run(**kwargs)

wf_runner.run(action, **kwargs)
wf_runner.run(**kwargs)

if post_wfile:
post_wf = Workflow(post_wfile)
pre_wf_runner = WorkflowRunner(post_wf)
pre_wf_runner.run(action, **kwargs)
pre_wf_runner.run(**kwargs)

except SystemExit as e:
if (e.code != 0) and on_failure:
kwargs['skip'] = list()
action = on_failure
wf_runner.run(action, **kwargs)
kwargs['action'] = on_failure
wf_runner.run(**kwargs)
else:
raise

if action:
log.info('Action "{}" finished successfully.'.format(action))
if kwargs['action']:
log.info('Action "{}" finished successfully.'.format(kwargs['action']))
else:
log.info('Workflow "{}" finished successfully.'.format(wfile))

Expand Down