From bc80f0983fe7249496485f321f33f0ed2165a3d6 Mon Sep 17 00:00:00 2001 From: KevinHock Date: Fri, 4 Jan 2019 11:03:42 -0800 Subject: [PATCH 1/4] =?UTF-8?q?=F0=9F=94=AD[Keyword=20Plugin]=20Add=20`nul?= =?UTF-8?q?l`=20to=20FALSE=5FPOSITIVES?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- detect_secrets/plugins/keyword.py | 1 + 1 file changed, 1 insertion(+) diff --git a/detect_secrets/plugins/keyword.py b/detect_secrets/plugins/keyword.py index f23713fe7..1301f4b02 100644 --- a/detect_secrets/plugins/keyword.py +++ b/detect_secrets/plugins/keyword.py @@ -65,6 +65,7 @@ 'none,', 'none}', 'not', + 'null', 'null,', 'password)', 'password,', From 17654e7a8f97754f3984970abc24c21708e24421 Mon Sep 17 00:00:00 2001 From: Aaron Loo Date: Fri, 4 Jan 2019 11:46:48 -0800 Subject: [PATCH 2/4] moving plugins/core to plugins/common; moving filetype determination to common code --- detect_secrets/core/audit.py | 4 +-- detect_secrets/core/secrets_collection.py | 2 +- detect_secrets/main.py | 2 +- detect_secrets/plugins/base.py | 2 +- .../plugins/{core => common}/__init__.py | 0 .../plugins/{core => common}/constants.py | 1 + detect_secrets/plugins/common/filetype.py | 23 +++++++++++++++++ .../{core => common}/ini_file_parser.py | 0 .../plugins/{core => common}/initialize.py | 0 .../{core => common}/yaml_file_parser.py | 2 +- .../plugins/high_entropy_strings.py | 4 +-- detect_secrets/plugins/keyword.py | 25 ++----------------- detect_secrets/pre_commit_hook.py | 2 +- .../{core => common}/initialize_test.py | 2 +- .../{core => common}/yaml_file_parser_test.py | 2 +- 15 files changed, 37 insertions(+), 34 deletions(-) rename detect_secrets/plugins/{core => common}/__init__.py (100%) rename detect_secrets/plugins/{core => common}/constants.py (99%) create mode 100644 detect_secrets/plugins/common/filetype.py rename detect_secrets/plugins/{core => common}/ini_file_parser.py (100%) rename detect_secrets/plugins/{core => common}/initialize.py (100%) rename detect_secrets/plugins/{core => common}/yaml_file_parser.py (98%) rename tests/plugins/{core => common}/initialize_test.py (97%) rename tests/plugins/{core => common}/yaml_file_parser_test.py (86%) diff --git a/detect_secrets/core/audit.py b/detect_secrets/core/audit.py index 99c7982a3..2e971d572 100644 --- a/detect_secrets/core/audit.py +++ b/detect_secrets/core/audit.py @@ -7,9 +7,9 @@ from builtins import input from collections import defaultdict -from ..plugins.core import initialize +from ..plugins.common import initialize +from ..plugins.common.filetype import determine_file_type from ..plugins.high_entropy_strings import HighEntropyStringsPlugin -from ..plugins.keyword import determine_file_type from ..plugins.keyword import KeywordDetector from .baseline import format_baseline_for_output from .baseline import merge_results diff --git a/detect_secrets/core/secrets_collection.py b/detect_secrets/core/secrets_collection.py index 227196c79..3bfb01f34 100644 --- a/detect_secrets/core/secrets_collection.py +++ b/detect_secrets/core/secrets_collection.py @@ -10,7 +10,7 @@ from detect_secrets import VERSION from detect_secrets.core.log import log from detect_secrets.core.potential_secret import PotentialSecret -from detect_secrets.plugins.core import initialize +from detect_secrets.plugins.common import initialize class SecretsCollection(object): diff --git a/detect_secrets/main.py b/detect_secrets/main.py index e29c6b6c0..5b7785dee 100644 --- a/detect_secrets/main.py +++ b/detect_secrets/main.py @@ -9,7 +9,7 @@ from detect_secrets.core import baseline from detect_secrets.core.log import log from detect_secrets.core.usage import ParserBuilder -from detect_secrets.plugins.core import initialize +from detect_secrets.plugins.common import initialize def parse_args(argv): diff --git a/detect_secrets/plugins/base.py b/detect_secrets/plugins/base.py index ee6aa4688..983d32351 100644 --- a/detect_secrets/plugins/base.py +++ b/detect_secrets/plugins/base.py @@ -3,7 +3,7 @@ from abc import abstractproperty from detect_secrets.core.potential_secret import PotentialSecret -from detect_secrets.plugins.core.constants import WHITELIST_REGEXES +from detect_secrets.plugins.common.constants import WHITELIST_REGEXES class BasePlugin(object): diff --git a/detect_secrets/plugins/core/__init__.py b/detect_secrets/plugins/common/__init__.py similarity index 100% rename from detect_secrets/plugins/core/__init__.py rename to detect_secrets/plugins/common/__init__.py diff --git a/detect_secrets/plugins/core/constants.py b/detect_secrets/plugins/common/constants.py similarity index 99% rename from detect_secrets/plugins/core/constants.py rename to detect_secrets/plugins/common/constants.py index 37725368b..6b35703c5 100644 --- a/detect_secrets/plugins/core/constants.py +++ b/detect_secrets/plugins/common/constants.py @@ -1,5 +1,6 @@ import re + WHITELIST_REGEXES = [ re.compile(r) for r in [ diff --git a/detect_secrets/plugins/common/filetype.py b/detect_secrets/plugins/common/filetype.py new file mode 100644 index 000000000..55032b1f2 --- /dev/null +++ b/detect_secrets/plugins/common/filetype.py @@ -0,0 +1,23 @@ +from enum import Enum + + +class FileType(Enum): + JAVASCRIPT = 0 + PHP = 1 + PYTHON = 2 + OTHER = 3 + + +def determine_file_type(filename): + """ + :param filename: str + + :rtype: FileType + """ + if filename.endswith('.js'): + return FileType.JAVASCRIPT + elif filename.endswith('.py'): + return FileType.PYTHON + elif filename.endswith('.php'): + return FileType.PHP + return FileType.OTHER diff --git a/detect_secrets/plugins/core/ini_file_parser.py b/detect_secrets/plugins/common/ini_file_parser.py similarity index 100% rename from detect_secrets/plugins/core/ini_file_parser.py rename to detect_secrets/plugins/common/ini_file_parser.py diff --git a/detect_secrets/plugins/core/initialize.py b/detect_secrets/plugins/common/initialize.py similarity index 100% rename from detect_secrets/plugins/core/initialize.py rename to detect_secrets/plugins/common/initialize.py diff --git a/detect_secrets/plugins/core/yaml_file_parser.py b/detect_secrets/plugins/common/yaml_file_parser.py similarity index 98% rename from detect_secrets/plugins/core/yaml_file_parser.py rename to detect_secrets/plugins/common/yaml_file_parser.py index 3a19d569a..2e6850a6d 100644 --- a/detect_secrets/plugins/core/yaml_file_parser.py +++ b/detect_secrets/plugins/common/yaml_file_parser.py @@ -1,6 +1,6 @@ import yaml -from detect_secrets.plugins.core.constants import WHITELIST_REGEX +from .constants import WHITELIST_REGEX class YamlFileParser(object): diff --git a/detect_secrets/plugins/high_entropy_strings.py b/detect_secrets/plugins/high_entropy_strings.py index 1ba4cb430..1bd546922 100644 --- a/detect_secrets/plugins/high_entropy_strings.py +++ b/detect_secrets/plugins/high_entropy_strings.py @@ -12,8 +12,8 @@ from .base import BasePlugin from detect_secrets.core.potential_secret import PotentialSecret -from detect_secrets.plugins.core.ini_file_parser import IniFileParser -from detect_secrets.plugins.core.yaml_file_parser import YamlFileParser +from detect_secrets.plugins.common.ini_file_parser import IniFileParser +from detect_secrets.plugins.common.yaml_file_parser import YamlFileParser IGNORED_SEQUENTIAL_STRINGS = ( diff --git a/detect_secrets/plugins/keyword.py b/detect_secrets/plugins/keyword.py index 1301f4b02..c50170ad0 100644 --- a/detect_secrets/plugins/keyword.py +++ b/detect_secrets/plugins/keyword.py @@ -27,9 +27,10 @@ from __future__ import absolute_import import re -from enum import Enum from .base import BasePlugin +from .common.filetype import determine_file_type +from .common.filetype import FileType from detect_secrets.core.potential_secret import PotentialSecret @@ -120,28 +121,6 @@ } -class FileType(Enum): - JAVASCRIPT = 0 - PHP = 1 - PYTHON = 2 - OTHER = 3 - - -def determine_file_type(filename): - """ - :param filename: str - - :rtype: FileType - """ - if filename.endswith('.js'): - return FileType.JAVASCRIPT - elif filename.endswith('.py'): - return FileType.PYTHON - elif filename.endswith('.php'): - return FileType.PHP - return FileType.OTHER - - class KeywordDetector(BasePlugin): """This checks if blacklisted keywords are present in the analyzed string. diff --git a/detect_secrets/pre_commit_hook.py b/detect_secrets/pre_commit_hook.py index f8ed92a3b..69dd99041 100644 --- a/detect_secrets/pre_commit_hook.py +++ b/detect_secrets/pre_commit_hook.py @@ -11,7 +11,7 @@ from detect_secrets.core.log import get_logger from detect_secrets.core.secrets_collection import SecretsCollection from detect_secrets.core.usage import ParserBuilder -from detect_secrets.plugins.core import initialize +from detect_secrets.plugins.common import initialize log = get_logger(format_string='%(message)s') diff --git a/tests/plugins/core/initialize_test.py b/tests/plugins/common/initialize_test.py similarity index 97% rename from tests/plugins/core/initialize_test.py rename to tests/plugins/common/initialize_test.py index 0315b716b..7b5161b06 100644 --- a/tests/plugins/core/initialize_test.py +++ b/tests/plugins/common/initialize_test.py @@ -3,7 +3,7 @@ import mock import pytest -from detect_secrets.plugins.core import initialize +from detect_secrets.plugins.common import initialize from detect_secrets.plugins.high_entropy_strings import Base64HighEntropyString from detect_secrets.plugins.high_entropy_strings import HexHighEntropyString diff --git a/tests/plugins/core/yaml_file_parser_test.py b/tests/plugins/common/yaml_file_parser_test.py similarity index 86% rename from tests/plugins/core/yaml_file_parser_test.py rename to tests/plugins/common/yaml_file_parser_test.py index f31f6dd0d..bfc65e26e 100644 --- a/tests/plugins/core/yaml_file_parser_test.py +++ b/tests/plugins/common/yaml_file_parser_test.py @@ -1,7 +1,7 @@ from __future__ import absolute_import from __future__ import unicode_literals -from detect_secrets.plugins.core.yaml_file_parser import YamlFileParser +from detect_secrets.plugins.common.yaml_file_parser import YamlFileParser from testing.mocks import mock_file_object From a599f89390e17d09b0e17a52641aebaf2b8c8c0e Mon Sep 17 00:00:00 2001 From: Aaron Loo Date: Fri, 4 Jan 2019 12:14:21 -0800 Subject: [PATCH 3/4] nicer secret_generator --- Makefile | 3 +++ detect_secrets/core/audit.py | 9 ++------- detect_secrets/plugins/base.py | 4 ++-- detect_secrets/plugins/high_entropy_strings.py | 2 +- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index 46467c20d..74049f23c 100644 --- a/Makefile +++ b/Makefile @@ -17,5 +17,8 @@ test: clean: find -name '*.pyc' -delete find -name '__pycache__' -delete + +.PHONY: super-clean +super-clean: clean rm -rf .tox rm -rf venv diff --git a/detect_secrets/core/audit.py b/detect_secrets/core/audit.py index 2e971d572..c61e095b8 100644 --- a/detect_secrets/core/audit.py +++ b/detect_secrets/core/audit.py @@ -10,7 +10,6 @@ from ..plugins.common import initialize from ..plugins.common.filetype import determine_file_type from ..plugins.high_entropy_strings import HighEntropyStringsPlugin -from ..plugins.keyword import KeywordDetector from .baseline import format_baseline_for_output from .baseline import merge_results from .bidirectional_iterator import BidirectionalIterator @@ -586,12 +585,8 @@ def _highlight_secret( def _raw_secret_generator(plugin, secret_line, filetype): """Generates raw secrets by re-scanning the line, with the specified plugin""" - if isinstance(plugin, KeywordDetector): - for raw_secret in plugin.secret_generator(secret_line, filetype=filetype): - yield raw_secret - else: - for raw_secret in plugin.secret_generator(secret_line): - yield raw_secret + for raw_secret in plugin.secret_generator(secret_line, filetype=filetype): + yield raw_secret if issubclass(plugin.__class__, HighEntropyStringsPlugin): with plugin.non_quoted_string_regex(strict=False): diff --git a/detect_secrets/plugins/base.py b/detect_secrets/plugins/base.py index 983d32351..d9a9cf154 100644 --- a/detect_secrets/plugins/base.py +++ b/detect_secrets/plugins/base.py @@ -47,7 +47,7 @@ def analyze_string(self, string, line_num, filename): raise NotImplementedError @abstractmethod - def secret_generator(self, string): + def secret_generator(self, string, *args, **kwargs): """Flags secrets in a given string, and yields the raw secret value. Used in self.analyze_string for PotentialSecret creation. @@ -127,7 +127,7 @@ def analyze_string(self, string, line_num, filename): return output - def secret_generator(self, string): + def secret_generator(self, string, *args, **kwargs): for regex in self.blacklist: for match in regex.findall(string): yield match diff --git a/detect_secrets/plugins/high_entropy_strings.py b/detect_secrets/plugins/high_entropy_strings.py index 1bd546922..10a2786b9 100644 --- a/detect_secrets/plugins/high_entropy_strings.py +++ b/detect_secrets/plugins/high_entropy_strings.py @@ -109,7 +109,7 @@ def analyze_string(self, string, line_num, filename): return output - def secret_generator(self, string): + def secret_generator(self, string, *args, **kwargs): # There may be multiple strings on the same line results = self.regex.findall(string) for result in results: From e1c96775257db69a3e829e83f6af958f410d2880 Mon Sep 17 00:00:00 2001 From: Aaron Loo Date: Fri, 4 Jan 2019 13:17:42 -0800 Subject: [PATCH 4/4] better examples --- test_data/short_files/first_line.php | 2 +- tests/main_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test_data/short_files/first_line.php b/test_data/short_files/first_line.php index 9b1d12e67..4ae03fef9 100644 --- a/test_data/short_files/first_line.php +++ b/test_data/short_files/first_line.php @@ -1,4 +1,4 @@ -seecret = 'BEEF0123456789a' +secret = 'notHighEnoughEntropy' skipped_sequential_false_positive = '0123456789a' print('second line') var = 'third line' diff --git a/tests/main_test.py b/tests/main_test.py index 160ab5739..ba122c922 100644 --- a/tests/main_test.py +++ b/tests/main_test.py @@ -170,7 +170,7 @@ def test_old_baseline_ignored_with_update_flag( ( 'test_data/short_files/first_line.php', textwrap.dedent(""" - 1:seecret = 'BEEF0123456789a' + 1:secret = 'notHighEnoughEntropy' 2:skipped_sequential_false_positive = '0123456789a' 3:print('second line') 4:var = 'third line'