From e319a51cde4a4f3b69a40878fdd022c099ba1d2b Mon Sep 17 00:00:00 2001 From: mayuriesha <89418259+mayuriesha@users.noreply.github.com> Date: Fri, 5 Nov 2021 14:17:56 -0700 Subject: [PATCH] Dropping support for --json and --compact and adding support for --output-format (#266) * Dropping support for --json and --compact and adding support for --output-format * Addressing review comments Co-authored-by: Joey Wilhelm --- CHANGELOG.md | 2 ++ README.md | 8 ++++++-- docs/source/examplecleanup.rst | 2 +- tartufo/cli.py | 24 ++++++++++++++++-------- tartufo/types.py | 12 ++++++++---- tartufo/util.py | 9 +++++---- tests/test_cli.py | 20 ++++++++++++++++++-- tests/test_util.py | 16 +++++++++------- 8 files changed, 65 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 437fab09..b543a80d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,8 @@ Features: thanks to @dclayton-godaddy for showing the way. * [#244](https://github.com/godaddy/tartufo/pull/244) - Drops support for `--fetch/--no-fetch` option for local scans +* [#253](https://github.com/godaddy/tartufo/issues/253) - Drops support for `--json` and `--compact` + and consolidates the two options into one `---output-format json/compact/text` * [#259](https://github.com/godaddy/tartufo/pull/259) - Adds a new `--scan-filenames/--no-scan-filenames` flag which allows users to enable or disable file name scanning. * [#254](https://github.com/godaddy/tartufo/pull/260) - Changes the default value of diff --git a/README.md b/README.md index 6ba2402c..302e3026 100644 --- a/README.md +++ b/README.md @@ -45,7 +45,6 @@ Usage: tartufo [OPTIONS] COMMAND [ARGS]... commit hook. Options: - --json / --no-json Output in JSON format. --rules FILENAME Path(s) to regex rules json list file(s). --default-regexes / --no-default-regexes Whether to include the default regex list @@ -53,7 +52,6 @@ Options: applicable if --rules is also specified. [default: True] - --compact / --no-compact Enable reduced output. [default: False] --entropy / --no-entropy Enable entropy checks. [default: True] --regex / --no-regex Enable high signal regexes checks. [default: False] @@ -122,6 +120,12 @@ Options: with keeping the results of individual runs of tartufo separated. + -of, --output-format TEXT Specify the format in which the output needs + to be generated `--output-format json/compact/text`. + Either `json`, `compact` or `text` can be specified. + If not provided (default) the output will be generated + in `text` format. + --git-rules-repo TEXT A file path, or git URL, pointing to a git repository containing regex rules to be used for scanning. By default, all .json files diff --git a/docs/source/examplecleanup.rst b/docs/source/examplecleanup.rst index 4404861e..d6b11867 100644 --- a/docs/source/examplecleanup.rst +++ b/docs/source/examplecleanup.rst @@ -46,7 +46,7 @@ More on this later!) .. code-block:: console # Run Tartufo on your repo and create a list of high entropy items to remove: - tartufo --regex --json scan-local-repo ${GITHUBREPO} | \ + tartufo --regex --output-format json scan-local-repo ${GITHUBREPO} | \ jq -r '.found_issues[].matched_string' | \ sort -u > remove.txt diff --git a/tartufo/cli.py b/tartufo/cli.py index 38d25cd3..9d825816 100644 --- a/tartufo/cli.py +++ b/tartufo/cli.py @@ -45,7 +45,6 @@ def get_command(self, ctx: click.Context, cmd_name: str) -> Optional[click.Comma name="tartufo", context_settings=dict(help_option_names=["-h", "--help"]), ) -@click.option("--json/--no-json", help="Output in JSON format.", is_flag=True) @click.option( "--rules", multiple=True, @@ -60,13 +59,6 @@ def get_command(self, ctx: click.Context, cmd_name: str) -> Optional[click.Comma help="Whether to include the default regex list when configuring" " search patterns. Only applicable if --rules is also specified.", ) -@click.option( - "--compact/--no-compact", - is_flag=True, - default=False, - show_default=True, - help="Enable reduced output.", -) @click.option( "--entropy/--no-entropy", is_flag=True, @@ -129,6 +121,22 @@ def get_command(self, ctx: click.Context, cmd_name: str) -> Optional[click.Comma are excluded unless effectively excluded via the --include-path-patterns option.""", ) +@click.option( + "-of", + "--output-format", + type=click.Choice( + [ + types.OutputFormat.Json.value, + types.OutputFormat.Compact.value, + types.OutputFormat.Text.value, + ] + ), + default="text", + help="""Specify the format in which the output needs to be generated + `--output-format json/compact/text`. Either `json`, `compact` or `text` + can be specified. If not provided (default) the output will be generated + in `text` format.""", +) @click.option( "-xe", "--exclude-entropy-patterns", diff --git a/tartufo/types.py b/tartufo/types.py index 18920070..b5aa3e1d 100644 --- a/tartufo/types.py +++ b/tartufo/types.py @@ -7,7 +7,6 @@ @dataclass class GlobalOptions: __slots__ = ( - "json", "rules", "default_regexes", "entropy", @@ -26,11 +25,10 @@ class GlobalOptions: "verbose", "quiet", "log_timestamps", - "compact", + "output_format", "b64_entropy_score", "hex_entropy_score", ) - json: bool rules: Tuple[TextIO, ...] default_regexes: bool entropy: bool @@ -49,7 +47,7 @@ class GlobalOptions: verbose: int quiet: bool log_timestamps: bool - compact: bool + output_format: Optional[str] b64_entropy_score: float hex_entropy_score: float @@ -97,6 +95,12 @@ class LogLevel(enum.IntEnum): DEBUG = 3 +class OutputFormat(enum.Enum): + Text = "text" # pylint: disable=invalid-name + Json = "json" # pylint: disable=invalid-name + Compact = "compact" # pylint: disable=invalid-name + + class TartufoException(Exception): """Base class for all package exceptions""" diff --git a/tartufo/util.py b/tartufo/util.py index d2f1bda0..670030f7 100644 --- a/tartufo/util.py +++ b/tartufo/util.py @@ -71,7 +71,7 @@ def echo_result( """ now = datetime.now().isoformat("T", "microseconds") - if options.json: + if options.output_format == types.OutputFormat.Json.value: output = { "scan_time": now, "project_path": repo_path, @@ -97,11 +97,12 @@ def echo_result( click.echo(f'{static_part[:-1]}, "found_issues": [', nl=False) delimiter = "" for issue in scanner.scan(): - live_part = json.dumps(issue.as_dict(compact=options.compact)) + compact = options.output_format == types.OutputFormat.Compact.value + live_part = json.dumps(issue.as_dict(compact=compact)) click.echo(f"{delimiter}{live_part}", nl=False) delimiter = ", " click.echo("]}") - elif options.compact: + elif options.output_format == types.OutputFormat.Compact.value: for issue in scanner.scan(): click.echo( f"[{issue.issue_type.value}] {issue.chunk.file_path}: {issue.matched_string} " @@ -261,5 +262,5 @@ def process_issues( echo_result(options, scan, repo_path, output_dir) if output_dir: write_outputs(scan.issues, output_dir) - if not options.json: + if options.output_format != types.OutputFormat.Json.value: click.echo(f"Results have been saved in {output_dir}") diff --git a/tests/test_cli.py b/tests/test_cli.py index 13f3da73..7cd9af55 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -155,7 +155,15 @@ def test_output_dir_is_not_called_out_when_outputting_json( runner = CliRunner() with runner.isolated_filesystem(): result = runner.invoke( - cli.main, ["--output-dir", "./foo", "--json", "scan-local-repo", "."] + cli.main, + [ + "--output-dir", + "./foo", + "--output-format", + "json", + "scan-local-repo", + ".", + ], ) # All other outputs are mocked, so this is ensuring that the # "Results have been saved in ..." message is not output. @@ -175,7 +183,15 @@ def test_output_dir_is_created_if_it_does_not_exist( runner = CliRunner() with runner.isolated_filesystem(): runner.invoke( - cli.main, ["--output-dir", "./foo", "--json", "scan-local-repo", "."] + cli.main, + [ + "--output-dir", + "./foo", + "--output-format", + "json", + "scan-local-repo", + ".", + ], ) self.assertTrue(Path("./foo").exists()) diff --git a/tests/test_util.py b/tests/test_util.py index cf967d63..87dfff01 100644 --- a/tests/test_util.py +++ b/tests/test_util.py @@ -110,7 +110,7 @@ class OutputTests(unittest.TestCase): @mock.patch("tartufo.scanner.ScannerBase") @mock.patch("tartufo.util.click") def test_echo_result_echos_all_when_not_json(self, mock_click, mock_scanner): - options = generate_options(GlobalOptions, json=False, verbose=0) + options = generate_options(GlobalOptions, verbose=0) mock_scanner.exclude_signatures = [] mock_scanner.scan.return_value = (1, 2, 3, 4) util.echo_result(options, mock_scanner, "", "") @@ -128,7 +128,7 @@ def test_echo_result_echos_all_when_not_json(self, mock_click, mock_scanner): @mock.patch("tartufo.scanner.ScannerBase") @mock.patch("tartufo.util.click") def test_echo_result_outputs_compact_format(self, mock_click, mock_scanner): - options = generate_options(GlobalOptions, json=False, verbose=0, compact=True) + options = generate_options(GlobalOptions, verbose=0, output_format="compact") issue1 = scanner.Issue( types.IssueType.Entropy, "foo", types.Chunk("fullfoobar", "/what/foo", {}) ) @@ -157,7 +157,7 @@ def test_echo_result_echos_message_when_clean( self, mock_time, mock_click, mock_scanner ): mock_time.now.return_value.isoformat.return_value = "now:now:now" - options = generate_options(GlobalOptions, json=False, quiet=False, verbose=0) + options = generate_options(GlobalOptions, quiet=False, verbose=0) mock_scanner.exclude_signatures = [] mock_scanner.issue_count = 0 mock_scanner.issues = [] @@ -183,7 +183,6 @@ def test_echo_result_echos_exclusions_verbose( ] options = generate_options( GlobalOptions, - json=False, quiet=False, verbose=1, exclude_signatures=exclude_signatures, @@ -212,7 +211,7 @@ def test_echo_result_echos_exclusions_verbose( @mock.patch("tartufo.scanner.ScannerBase") @mock.patch("tartufo.util.click") def test_echo_result_echos_no_message_when_quiet(self, mock_click, mock_scanner): - options = generate_options(GlobalOptions, json=False, quiet=True, verbose=0) + options = generate_options(GlobalOptions, quiet=True, verbose=0) mock_scanner.issues = [] mock_scanner.exclude_signatures = [] util.echo_result(options, mock_scanner, "", "") @@ -235,7 +234,10 @@ def test_echo_result_outputs_proper_json_when_requested( mock_scanner.scan.return_value = (issue_1, issue_2) mock_scanner.excluded_paths = [] options = generate_options( - GlobalOptions, json=True, exclude_signatures=[], exclude_entropy_patterns=[] + GlobalOptions, + output_format=types.OutputFormat.Json.value, + exclude_signatures=[], + exclude_entropy_patterns=[], ) # We're generating JSON piecemeal, so if we want to be safe we'll recover @@ -302,7 +304,7 @@ def test_echo_result_outputs_proper_json_when_requested_pathtype( ] options = generate_options( GlobalOptions, - json=True, + output_format=types.OutputFormat.Json.value, exclude_signatures=exclude_signatures, exclude_entropy_patterns=exclude_entropy_patterns, )