Reorganization

Makefile

@@ -17,5 +17,8 @@ test:
 clean:
 	find -name '*.pyc' -delete
 	find -name '__pycache__' -delete
+
+.PHONY: super-clean
+super-clean: clean
 	rm -rf .tox
 	rm -rf venv

detect_secrets/core/audit.py

@@ -7,10 +7,9 @@
 from builtins import input
 from collections import defaultdict
 
-from ..plugins.core import initialize
+from ..plugins.common import initialize
+from ..plugins.common.filetype import determine_file_type
 from ..plugins.high_entropy_strings import HighEntropyStringsPlugin
-from ..plugins.keyword import determine_file_type
-from ..plugins.keyword import KeywordDetector
 from .baseline import format_baseline_for_output
 from .baseline import merge_results
 from .bidirectional_iterator import BidirectionalIterator
@@ -586,12 +585,8 @@ def _highlight_secret(
 
 def _raw_secret_generator(plugin, secret_line, filetype):
     """Generates raw secrets by re-scanning the line, with the specified plugin"""
-    if isinstance(plugin, KeywordDetector):
-        for raw_secret in plugin.secret_generator(secret_line, filetype=filetype):
-            yield raw_secret
-    else:
-        for raw_secret in plugin.secret_generator(secret_line):
-            yield raw_secret
+    for raw_secret in plugin.secret_generator(secret_line, filetype=filetype):
+        yield raw_secret
 
     if issubclass(plugin.__class__, HighEntropyStringsPlugin):
         with plugin.non_quoted_string_regex(strict=False):

detect_secrets/core/secrets_collection.py

@@ -10,7 +10,7 @@
 from detect_secrets import VERSION
 from detect_secrets.core.log import log
 from detect_secrets.core.potential_secret import PotentialSecret
-from detect_secrets.plugins.core import initialize
+from detect_secrets.plugins.common import initialize
 
 
 class SecretsCollection(object):

detect_secrets/main.py

@@ -9,7 +9,7 @@
 from detect_secrets.core import baseline
 from detect_secrets.core.log import log
 from detect_secrets.core.usage import ParserBuilder
-from detect_secrets.plugins.core import initialize
+from detect_secrets.plugins.common import initialize
 
 
 def parse_args(argv):

detect_secrets/plugins/base.py

@@ -3,7 +3,7 @@
 from abc import abstractproperty
 
 from detect_secrets.core.potential_secret import PotentialSecret
-from detect_secrets.plugins.core.constants import WHITELIST_REGEXES
+from detect_secrets.plugins.common.constants import WHITELIST_REGEXES
 
 
 class BasePlugin(object):
@@ -47,7 +47,7 @@ def analyze_string(self, string, line_num, filename):
         raise NotImplementedError
 
     @abstractmethod
-    def secret_generator(self, string):
+    def secret_generator(self, string, *args, **kwargs):
         """Flags secrets in a given string, and yields the raw secret value.
         Used in self.analyze_string for PotentialSecret creation.
 
@@ -127,7 +127,7 @@ def analyze_string(self, string, line_num, filename):
 
         return output
 
-    def secret_generator(self, string):
+    def secret_generator(self, string, *args, **kwargs):
         for regex in self.blacklist:
             for match in regex.findall(string):
                 yield match

detect_secrets/plugins/core/constants.py → detect_secrets/plugins/common/constants.py

@@ -1,5 +1,6 @@
 import re
 
+
 WHITELIST_REGEXES = [
     re.compile(r)
     for r in [

detect_secrets/plugins/common/filetype.py

@@ -0,0 +1,23 @@
+from enum import Enum
+
+
+class FileType(Enum):
+    JAVASCRIPT = 0
+    PHP = 1
+    PYTHON = 2
+    OTHER = 3
+
+
+def determine_file_type(filename):
+    """
+    :param filename: str
+
+    :rtype: FileType
+    """
+    if filename.endswith('.js'):
+        return FileType.JAVASCRIPT
+    elif filename.endswith('.py'):
+        return FileType.PYTHON
+    elif filename.endswith('.php'):
+        return FileType.PHP
+    return FileType.OTHER

detect_secrets/plugins/core/yaml_file_parser.py → detect_secrets/plugins/common/yaml_file_parser.py

@@ -1,6 +1,6 @@
 import yaml
 
-from detect_secrets.plugins.core.constants import WHITELIST_REGEX
+from .constants import WHITELIST_REGEX
 
 
 class YamlFileParser(object):

detect_secrets/plugins/high_entropy_strings.py

@@ -12,8 +12,8 @@
 
 from .base import BasePlugin
 from detect_secrets.core.potential_secret import PotentialSecret
-from detect_secrets.plugins.core.ini_file_parser import IniFileParser
-from detect_secrets.plugins.core.yaml_file_parser import YamlFileParser
+from detect_secrets.plugins.common.ini_file_parser import IniFileParser
+from detect_secrets.plugins.common.yaml_file_parser import YamlFileParser
 
 
 IGNORED_SEQUENTIAL_STRINGS = (
@@ -109,7 +109,7 @@ def analyze_string(self, string, line_num, filename):
 
         return output
 
-    def secret_generator(self, string):
+    def secret_generator(self, string, *args, **kwargs):
         # There may be multiple strings on the same line
         results = self.regex.findall(string)
         for result in results:

detect_secrets/plugins/keyword.py

@@ -27,9 +27,10 @@
 from __future__ import absolute_import
 
 import re
-from enum import Enum
 
 from .base import BasePlugin
+from .common.filetype import determine_file_type
+from .common.filetype import FileType
 from detect_secrets.core.potential_secret import PotentialSecret
 
 
@@ -65,6 +66,7 @@
     'none,',
     'none}',
     'not',
+    'null',
     'null,',
     'password)',
     'password,',
@@ -119,28 +121,6 @@
 }
 
 
-class FileType(Enum):
-    JAVASCRIPT = 0
-    PHP = 1
-    PYTHON = 2
-    OTHER = 3
-
-
-def determine_file_type(filename):
-    """
-    :param filename: str
-
-    :rtype: FileType
-    """
-    if filename.endswith('.js'):
-        return FileType.JAVASCRIPT
-    elif filename.endswith('.py'):
-        return FileType.PYTHON
-    elif filename.endswith('.php'):
-        return FileType.PHP
-    return FileType.OTHER
-
-
 class KeywordDetector(BasePlugin):
     """This checks if blacklisted keywords
     are present in the analyzed string.

detect_secrets/pre_commit_hook.py

@@ -11,7 +11,7 @@
 from detect_secrets.core.log import get_logger
 from detect_secrets.core.secrets_collection import SecretsCollection
 from detect_secrets.core.usage import ParserBuilder
-from detect_secrets.plugins.core import initialize
+from detect_secrets.plugins.common import initialize
 
 
 log = get_logger(format_string='%(message)s')

test_data/short_files/first_line.php

@@ -1,4 +1,4 @@
-seecret = 'BEEF0123456789a'
+secret = 'notHighEnoughEntropy'
 skipped_sequential_false_positive = '0123456789a'
 print('second line')
 var = 'third line'

tests/main_test.py

@@ -170,7 +170,7 @@ def test_old_baseline_ignored_with_update_flag(
             (
                 'test_data/short_files/first_line.php',
                 textwrap.dedent("""
-                    1:seecret = 'BEEF0123456789a'
+                    1:secret = 'notHighEnoughEntropy'
                     2:skipped_sequential_false_positive = '0123456789a'
                     3:print('second line')
                     4:var = 'third line'

tests/plugins/core/initialize_test.py → tests/plugins/common/initialize_test.py

@@ -3,7 +3,7 @@
 import mock
 import pytest
 
-from detect_secrets.plugins.core import initialize
+from detect_secrets.plugins.common import initialize
 from detect_secrets.plugins.high_entropy_strings import Base64HighEntropyString
 from detect_secrets.plugins.high_entropy_strings import HexHighEntropyString
 

tests/plugins/core/yaml_file_parser_test.py → tests/plugins/common/yaml_file_parser_test.py

@@ -1,7 +1,7 @@
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
-from detect_secrets.plugins.core.yaml_file_parser import YamlFileParser
+from detect_secrets.plugins.common.yaml_file_parser import YamlFileParser
 from testing.mocks import mock_file_object