WIP: Add json parameter to clean up stdout into valid JSON

Author: cognifloyd

actions/ansible.py

@@ -1,7 +1,11 @@
 #!/usr/bin/env python
 
+import json
+import os
+import shutil
 import sys
-from lib.ansible_base import AnsibleBaseRunner
+import tempfile
+from lib.ansible_base import AnsibleBaseRunner, ParamaterConflict
 
 __all__ = [
     'AnsibleRunner'
@@ -33,6 +37,45 @@ class AnsibleRunner(AnsibleBaseRunner):
 
     def __init__(self, *args, **kwargs):
         super(AnsibleRunner, self).__init__(*args, **kwargs)
+        self.tree_dir = None
+        self.one_line = False
+        if '--one_line' in args:
+            self.one_line = True
+
+    def handle_json_arg(self):
+        if next((True for arg in self.args if arg.startswith('--tree')), False):
+            msg = "--json uses --tree internally. Setting both --tree and --json is not supported."
+            raise ParamaterConflict(msg)
+        execution_id = os.environ.get('ST2_ACTION_EXECUTION_ID', 'EXECUTION_ID_NA')
+        self.tree_dir = tempfile.mkdtemp(prefix='{}.'.format(execution_id))
+
+        tree_arg = '--tree {}'.format(self.tree_dir)
+        self.args.append(tree_arg)
+        self.stdout = open(os.devnull, 'w')
+
+    def output_json(self):
+        output = {}
+        for host in os.listdir(self.tree_dir):
+            # one file per host in tree dir; name of host is name of file
+            with open(os.path.join(self.tree_dir, host), 'r') as host_output:
+                try:
+                    output[host] = json.load(host_output)
+                except ValueError:
+                    # something is messed up in the json, so include it as a string.
+                    host_output.seek(0)
+                    output[host] = host_output.read()
+        if self.one_line:
+            print(json.dumps(output))
+        else:
+            print(json.dumps(output, indent=2))
+
+    def cleanup(self):
+        shutil.rmtree(self.tree_dir)
+        self.stdout.close()
+
+    def post_execute(self):
+        self.output_json()
+        self.cleanup()
 
 
 if __name__ == '__main__':

actions/ansible_playbook.py

@@ -1,6 +1,12 @@
 #!/usr/bin/env python
 
+from __future__ import print_function
+
+import os
+import subprocess
+import re
 import sys
+
 from lib.ansible_base import AnsibleBaseRunner
 
 __all__ = [
@@ -38,6 +44,52 @@ class AnsiblePlaybookRunner(AnsibleBaseRunner):
     def __init__(self, *args, **kwargs):
         super(AnsiblePlaybookRunner, self).__init__(*args, **kwargs)
 
+    def handle_json_arg(self):
+        os.environ['ANSIBLE_CALLBAKC_PLUGIN'] = 'json'
+        self.stdout = subprocess.PIPE
+        # TODO: --json is probably not be compatible with other options
+        # like syntax-check, verbose, and possibly others
+
+    def popen_call(self, p):
+        """
+        :param subprocess.Popen p:
+        :return:
+        """
+        if not self.json_output:
+            return super(AnsiblePlaybookRunner, self).popen_call(p)
+
+        # lines that should go to stderr instead of stdout
+        stderr_lines = re.compile(
+            r'('
+
+            # these were identified in this closed PR:
+            #    https://github.com/ansible/ansible/pull/17448/files
+            # see https://github.com/ansible/ansible/issues/17122
+            r"^\tto retry, use: --limit @.*$"
+            r'|'
+            r"^skipping playbook include '.*' due to conditional test failure$"
+            r'|'
+            r"^statically included: .*$"
+
+            # other possibilities:
+            # r'|'
+            # r'^to see the full traceback, use -vvv$'
+            # r'|'
+            # r"^The full traceback was:$"
+            # but then I would somehow have to include the traceback as well...
+
+            r')'
+        )
+
+        while p.poll() is None:
+            line = p.stdout.readline()
+            if re.match(stderr_lines, line):
+                print(line, file=sys.stderr)
+            else:
+                print(line)
+
+        return p.returncode
+
 
 if __name__ == '__main__':
     AnsiblePlaybookRunner(sys.argv).execute()

actions/command.yaml

@@ -105,3 +105,6 @@ parameters:
   version:
     description: "Show ansible version number and exit"
     type: boolean
+  json:
+    description: "Clean up Ansible's output to ensure it is valid jSON"
+    type: boolean

actions/command_local.yaml

@@ -105,3 +105,6 @@ parameters:
   version:
     description: "Show ansible version number and exit"
     type: boolean
+  json:
+    description: "Clean up Ansible's output to ensure it is valid jSON"
+    type: boolean

actions/lib/ansible_base.py

@@ -23,7 +23,11 @@ def __init__(self, args):
         :param args: Input command line arguments
         :type args: ``list``
         """
+        self.json_output = False
+        self.stdout = None
+
         self.args = args[1:]
+        self._json_output_arg()
         self._parse_extra_vars()  # handle multiple entries in --extra_vars arg
         self._prepend_venv_path()
 
@@ -96,17 +100,41 @@ def _prepend_venv_path():
 
         os.environ['PATH'] = ':'.join(new_path)
 
+    def _json_output_arg(self):
+        for i, arg in enumerate(self.args):
+            if '--json' in arg and hasattr(self, 'handle_json_arg'):
+                self.json_output = True
+                self.handle_json_arg()
+                # if ansible-playbook, add env arg
+                del self.args[i]  # The json arg is a ST2 specific addition & should not pass on.
+                break
+            elif '--one_line' in arg:
+                self.one_line = True
+
+    def handle_json_arg(self):
+        pass
+
+    def popen_call(self, p):
+        return p.wait()
+
     def execute(self):
         """
         Execute the command and stream stdout and stderr output
         from child process as it appears without delay.
         Terminate with child's exit code.
         """
-        exit_code = subprocess.call(self.cmd, env=os.environ.copy())
+        p = subprocess.Popen(self.cmd, env=os.environ.copy(), stdout=self.stdout)
+        exit_code = self.popen_call(p)
         if exit_code is not 0:
             sys.stderr.write('Executed command "%s"\n' % ' '.join(self.cmd))
+
+        self.post_execute()
+
         sys.exit(exit_code)
 
+    def post_execute(self):
+        pass
+
     @property
     @shell.replace_args('REPLACEMENT_RULES')
     def cmd(self):
@@ -139,3 +167,7 @@ def binary(self):
             sys.exit(1)
 
         return binary_path
+
+
+class ParamaterConflict(Exception):
+    pass

actions/playbook.yaml

@@ -117,3 +117,6 @@ parameters:
   version:
     description: "Show ansible version number and exit"
     type: boolean
+  json:
+    description: "Clean up Ansible's output to ensure it is valid jSON"
+    type: boolean