Safety POST for pyup.io

safety/cli.py

@@ -5,6 +5,7 @@
 import logging
 import os
 import sys
+import tempfile
 
 import click
 
@@ -13,7 +14,7 @@
 from safety.errors import SafetyException, SafetyError
 from safety.formatter import SafetyFormatter
 from safety.output_utils import should_add_nl
-from safety.safety import get_packages, read_vulnerabilities
+from safety.safety import get_packages, read_vulnerabilities, fetch_policy, post_results
 from safety.util import get_proxy_dict, get_packages_licenses, output_exception, \
     MutuallyExclusiveOption, DependentOption, transform_ignore, SafetyPolicyFile, active_color_if_needed, \
     get_processed_options, get_safety_version, json_alias, bare_alias, SafetyContext
@@ -83,11 +84,16 @@ def cli(ctx, debug, telemetry, disable_optional_telemetry_data):
               help="Output standard exit codes. Default: --exit-code")
 @click.option("--policy-file", type=SafetyPolicyFile(), default='.safety-policy.yml',
               help="Define the policy file to be used")
+@click.option("--audit-and-monitor/--no-audit-and-monitor", default=True,
+              help="Send results back to pyup.io for viewing on your dashboard. Requires an API key.")
+@click.option("--project", default=None,
+              help="Project to associate this scan with on pyup.io. Defaults to a canonicalized github style name if available, otherwise unknown")
+
 @click.option("--save-json", default="", help="Path to where output file will be placed, if the path is a directory, "
                                               "Safety will use safety-report.json as filename. Default: empty")
 @click.pass_context
 def check(ctx, key, db, full_report, stdin, files, cache, ignore, output, json, bare, proxy_protocol, proxy_host, proxy_port,
-          exit_code, policy_file, save_json):
+          exit_code, policy_file, save_json, audit_and_monitor, project):
     """
     Find vulnerabilities in Python dependencies at the target provided.
 
@@ -103,13 +109,36 @@ def check(ctx, key, db, full_report, stdin, files, cache, ignore, output, json,
             LOG.info('Not local DB used, Getting announcements')
             announcements = safety.get_announcements(key=key, proxy=proxy_dictionary, telemetry=ctx.parent.telemetry)
 
+        if key:
+            server_policies = fetch_policy(key=key, proxy=proxy_dictionary)
+            server_audit_and_monitor = server_policies["audit_and_monitor"]
+            server_safety_policy = server_policies["safety_policy"]
+        else:
+            server_audit_and_monitor = False
+            server_safety_policy = ""
+
+        if server_safety_policy and policy_file:
+            click.secho(
+                "Warning: both a local policy file '{policy_filename}' and a server sent policy are present. "
+                "Continuing with the local policy file.".format(policy_filename=policy_file['filename']),
+                fg="yellow",
+                file=sys.stderr
+            )
+        elif server_safety_policy:
+            with tempfile.NamedTemporaryFile(prefix='server-safety-policy-') as tmp:
+                tmp.write(server_safety_policy.encode('utf-8'))
+                tmp.seek(0)
+
+                policy_file = SafetyPolicyFile().convert(tmp.name, param=None, ctx=None)
+                LOG.info('Using server side policy file')
+
         ignore_severity_rules = None
         ignore, ignore_severity_rules, exit_code = get_processed_options(policy_file, ignore,
                                                                          ignore_severity_rules, exit_code)
 
         is_env_scan = not stdin and not files
         params = {'stdin': stdin, 'files': files, 'policy_file': policy_file, 'continue_on_error': not exit_code,
-                  'ignore_severity_rules': ignore_severity_rules}
+                  'ignore_severity_rules': ignore_severity_rules, 'project': project, 'audit_and_monitor': server_audit_and_monitor and audit_and_monitor}
         LOG.info('Calling the check function')
         vulns, db_full = safety.check(packages=packages, key=key, db_mirror=db, cached=cache, ignore_vulns=ignore,
                                       ignore_severity_rules=ignore_severity_rules, proxy=proxy_dictionary,
@@ -121,8 +150,32 @@ def check(ctx, key, db, full_report, stdin, files, cache, ignore, output, json,
         LOG.info('Safety is going to calculate remediations')
         remediations = safety.calculate_remediations(vulns, db_full)
 
+        json_report = None
+        if save_json or (server_audit_and_monitor and audit_and_monitor):
+            default_name = 'safety-report.json'
+            json_report = SafetyFormatter(output='json').render_vulnerabilities(announcements, vulns, remediations,
+                                                                                full_report, packages)
+
+            if server_audit_and_monitor and audit_and_monitor:
+                policy_contents = ''
+                if policy_file:
+                    policy_contents = policy_file.get('raw', '')
+
+                r = post_results(key=key, proxy=proxy_dictionary, safety_json=json_report, policy_file=policy_contents)
+                SafetyContext().params['audit_and_monitor_url'] = r.get('url')
+
+            if save_json:
+                if os.path.isdir(save_json):
+                    save_json = os.path.join(save_json, default_name)
+
+                with open(save_json, 'w+') as output_json_file:
+                    output_json_file.write(json_report)
+
         LOG.info('Safety is going to render the vulnerabilities report using %s output', output)
-        output_report = SafetyFormatter(output=output).render_vulnerabilities(announcements, vulns, remediations,
+        if json_report and output == 'json':
+            output_report = json_report
+        else:
+            output_report = SafetyFormatter(output=output).render_vulnerabilities(announcements, vulns, remediations,
                                                                               full_report, packages)
 
         # Announcements are send to stderr if not terminal, it doesn't depend on "exit_code" value
@@ -134,19 +187,6 @@ def check(ctx, key, db, full_report, stdin, files, cache, ignore, output, json,
         LOG.info('Vulnerabilities found (Not ignored): %s', len(found_vulns))
         LOG.info('All vulnerabilities found (ignored and Not ignored): %s', len(vulns))
 
-        if save_json:
-            default_name = 'safety-report.json'
-            json_report = output_report
-
-            if output != 'json':
-                json_report = SafetyFormatter(output='json').render_vulnerabilities(announcements, vulns, remediations,
-                                                                                    full_report, packages)
-            if os.path.isdir(save_json):
-                save_json = os.path.join(save_json, default_name)
-
-            with open(save_json, 'w+') as output_json_file:
-                output_json_file.write(json_report)
-
         click.secho(output_report, nl=should_add_nl(output, found_vulns), file=sys.stdout)
 
         if exit_code and found_vulns:

safety/output_utils.py

@@ -6,7 +6,7 @@
 import click
 
 from safety.constants import RED, YELLOW
-from safety.util import get_safety_version, Package, get_terminal_size, SafetyContext
+from safety.util import get_safety_version, Package, get_terminal_size, SafetyContext, build_telemetry_data, build_git_data
 
 LOG = logging.getLogger(__name__)
 
@@ -570,13 +570,22 @@ def get_report_brief_info(as_dict=False, report_type=1, **kwargs):
     safety_policy_used = []
 
     brief_data['policy_file'] = policy_file.get('filename', '-') if policy_file else None
+    brief_data['policy_file_source'] = 'server' if brief_data['policy_file'] and 'server-safety-policy' in brief_data['policy_file'] else 'local'
 
     if policy_file and policy_file.get('filename', False):
         safety_policy_used = [
             {'style': False, 'value': '\nScanning using a security policy file'},
             {'style': True, 'value': ' {0}'.format(policy_file.get('filename', '-'))},
         ]
 
+    audit_and_monitor = []
+    if context.params.get('audit_and_monitor'):
+        logged_url = context.params.get('audit_and_monitor_url') if context.params.get('audit_and_monitor_url') else "https://pyup.io"
+        audit_and_monitor = [
+            {'style': False, 'value': '\nLogging scan results to'},
+            {'style': True, 'value': ' {0}'.format(logged_url)},
+        ]
+
     current_time = str(datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
 
     brief_data['api_key'] = bool(key)
@@ -612,6 +621,13 @@ def get_report_brief_info(as_dict=False, report_type=1, **kwargs):
              {'style': True, 'value': f' license {"type" if brief_data["licenses_found"] == 1 else "types"} found'}],
         ]
 
+    brief_data['telemetry'] = build_telemetry_data()
+
+    brief_data['git'] = build_git_data()
+    brief_data['project'] = context.params.get('project', None)
+
+    brief_data['json_version'] = 1
+
     using_sentence = build_using_sentence(key, db)
     scanned_count_sentence = build_scanned_count_sentence(packages)
 
@@ -621,7 +637,7 @@ def get_report_brief_info(as_dict=False, report_type=1, **kwargs):
      {'style': True, 'value': 'v' + get_safety_version()},
      {'style': False, 'value': ' is scanning for '},
      {'style': True, 'value': scanning_types.get(context.command, {}).get('name', '')},
-     {'style': True, 'value': '...'}] + safety_policy_used, action_executed
+     {'style': True, 'value': '...'}] + safety_policy_used + audit_and_monitor, action_executed
      ] + [nl] + scanned_items + [nl] + [using_sentence] + [scanned_count_sentence] + [timestamp]
 
     brief_info.extend(additional_data)

safety/safety.py

@@ -142,6 +142,63 @@ def fetch_database_url(mirror, db_name, key, cached, proxy, telemetry=True):
     return data
 
 
+def fetch_policy(key, proxy):
+    url = f"{API_BASE_URL}policy/"
+    headers = {"X-Api-Key": key}
+
+    if not proxy:
+        proxy = {}
+
+    try:
+        LOG.debug(f'Getting policy')
+        r = session.get(url=url, timeout=REQUEST_TIMEOUT, headers=headers, proxies=proxy)
+        LOG.debug(r.text)
+        return r.json()
+    except:
+        import click
+
+        LOG.exception("Error fetching policy")
+        click.secho(
+            "Warning: couldn't fetch policy from pyup.io.",
+            fg="yellow",
+            file=sys.stderr
+        )
+
+        return {"safety_policy": "", "audit_and_monitor": False}
+
+
+def post_results(key, proxy, safety_json, policy_file):
+    url = f"{API_BASE_URL}result/"
+    headers = {"X-Api-Key": key}
+
+    if not proxy:
+        proxy = {}
+
+    # safety_json is in text form already. policy_file is a text YAML
+    audit_report = {
+        "safety_json": json.loads(safety_json),
+        "policy_file": policy_file
+    }
+
+    try:
+        LOG.debug(f'Posting results: {audit_report}')
+        r = session.post(url=url, timeout=REQUEST_TIMEOUT, headers=headers, proxies=proxy, json=audit_report)
+        LOG.debug(r.text)
+
+        return r.json()
+    except:
+        import click
+
+        LOG.exception("Error posting results")
+        click.secho(
+            "Warning: couldn't upload results to pyup.io.",
+            fg="yellow",
+            file=sys.stderr
+        )
+
+        return {}
+
+
 def fetch_database_file(path, db_name):
     full_path = os.path.join(path, db_name)
     if not os.path.exists(full_path):
@@ -252,7 +309,7 @@ def ignore_vuln_if_needed(vuln_id, cve, ignore_vulns, ignore_severity_rules):
 
 @sync_safety_context
 def check(packages, key=False, db_mirror=False, cached=0, ignore_vulns=None, ignore_severity_rules=None, proxy=None,
-          include_ignored=False, is_env_scan=True, telemetry=True, params=None):
+          include_ignored=False, is_env_scan=True, telemetry=True, params=None, project=None):
     SafetyContext().command = 'check'
     db = fetch_database(key=key, db=db_mirror, cached=cached, proxy=proxy, telemetry=telemetry)
     db_full = None

safety/util.py

@@ -1,6 +1,7 @@
 import logging
 import os
 import platform
+
 import sys
 from datetime import datetime
 from difflib import SequenceMatcher
@@ -14,6 +15,7 @@
 from packaging.version import parse as parse_version
 from ruamel.yaml import YAML
 from ruamel.yaml.error import MarkedYAMLError
+import safety
 
 from safety.constants import EXIT_CODE_FAILURE, EXIT_CODE_OK
 from safety.models import Package, RequirementFile
@@ -208,6 +210,39 @@ def build_telemetry_data(telemetry=True):
     return body
 
 
+def build_git_data():
+    import subprocess
+
+    is_git = subprocess.run(["git", "rev-parse", "--is-inside-work-tree"], stdout=subprocess.PIPE).stdout.decode('utf-8').strip()
+
+    if is_git == "true":
+        result = {
+            "branch": "",
+            "tag": "",
+            "commit": "",
+            "dirty": "",
+            "origin": ""
+        }
+
+        try:
+            result['branch'] = subprocess.run(["git", "symbolic-ref", "--short", "-q", "HEAD"], stdout=subprocess.PIPE).stdout.decode('utf-8').strip()
+            result['tag'] = subprocess.run(["git", "describe", "--tags", "--exact-match"], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).stdout.decode('utf-8').strip()
+
+            commit = subprocess.run(["git", "describe", '--match=""', '--always', '--abbrev=40', '--dirty'], stdout=subprocess.PIPE).stdout.decode('utf-8').strip()
+            result['dirty'] = commit.endswith('-dirty')
+            result['commit'] = commit.split("-dirty")[0]
+
+            result['origin'] = subprocess.run(["git", "remote", "get-url", "origin"], stdout=subprocess.PIPE).stdout.decode('utf-8').strip()
+        except Exception:
+            pass
+
+        return result
+    else:
+        return {
+            "error": "not-git-repo"
+        }
+
+
 def output_exception(exception, exit_code_output=True):
     click.secho(str(exception), fg="red", file=sys.stderr)
 
@@ -427,8 +462,9 @@ def convert(self, value, param, ctx):
             filename = ''
 
             try:
+                raw = f.read()
                 yaml = YAML(typ='safe', pure=False)
-                safety_policy = yaml.load(f)
+                safety_policy = yaml.load(raw)
                 filename = f.name
             except Exception as e:
                 show_parsed_hint = isinstance(e, MarkedYAMLError)
@@ -518,6 +554,7 @@ def convert(self, value, param, ctx):
 
                 safety_policy['security']['ignore-vulnerabilities'] = normalized
                 safety_policy['filename'] = filename
+                safety_policy['raw'] = raw
             else:
                 safety_policy['security']['ignore-vulnerabilities'] = {}