Merge pull request #286 from SAP/develop

Upgrade to v4.12

.github/dependabot.yml

@@ -9,3 +9,5 @@ updates:
     directory: "/" # Location of package manifests
     schedule:
       interval: "weekly"
+    # Open pull requests for version updates on the `develop` branch
+    target-branch: "develop"

.github/workflows/test.yml

@@ -10,8 +10,7 @@ jobs:
 
     strategy:
       matrix:
-        #python-version: ['3.7', '3.8', '3.9', '3.10']
-        python-version: ['3.7', '3.8', '3.9']
+        python-version: ['3.8', '3.9', '3.10', '3.11']
 
     services:
       postgres:
@@ -44,7 +43,7 @@ jobs:
 
     - name: Install OS dependencies
       run: |
-        sudo apt install -y build-essential python3-dev libhyperscan-dev
+        sudo apt install -y build-essential python3-dev
 
     - name: Cache python dependencies
       uses: actions/cache@v3
@@ -54,14 +53,13 @@ jobs:
         restore-keys: |
           ${{ runner.os }}-pip-
 
-    - name: Install python dependencies
+    - name: Install credentialdigger
       run: |
-        pip install -r ./requirements.txt
-        pip install -r ./tests/tests-requirements.txt
+        pip install . --user
 
-    - name: Install credentialdigger
+    - name: Install tests dependencies
       run: |
-        python setup.py install --user
+        pip install -r ./tests/tests-requirements.txt
 
     - name: Run unit tests
       run: |

README.md

@@ -47,10 +47,10 @@ The goal of Credential Digger is to reduce the amount of false positive data on
 
 
 The tool supports several scan flavors: public and private repositories on
-github and gitlab, wiki pages, github organizations, local git repositories, local files and folders.
+github and gitlab, pull requests, wiki pages, github organizations, local git repositories, local files and folders.
 Please refer to the [Wiki](https://github.com/SAP/credential-digger/wiki) for the complete documentation.
 
-For the complete description of the approach of Credential Digger, [you can read this publication](https://www.scitepress.org/Papers/2021/102381/102381.pdf).
+For the complete description of the approach of Credential Digger (versions <4.4), [you can read this publication](https://www.scitepress.org/Papers/2021/102381/102381.pdf).
 
 ```
 @InProceedings {lrnto-icissp21,
@@ -65,22 +65,16 @@ For the complete description of the approach of Credential Digger, [you can read
 
 ## Requirements
 
-Credential Digger supports Python >= 3.6 and < 3.10, and works only with Linux and MacOS systems.
+Credential Digger supports Python >= 3.8 and < 3.12, and works only with Linux and MacOS systems.
 In case you don't meet these requirements, you may consider running a [Docker container](#docker) (that also includes a user interface).
 
 
 ## Download and Installation
 
-First, you need to install the regular expression matching library [Hyperscan](https://github.com/intel/hyperscan). Be sure to have `build-essential` and `python3-dev` too.
+First, you need to install some dependencies (namely, `build-essential` and `python3-dev`). No need to explicitely install hyperscan anymore.
 
 ```bash
-sudo apt install -y libhyperscan-dev build-essential python3-dev
-```
-
-or (for MacOS):
-
-```bash
-brew install hyperscan
+sudo apt install -y build-essential python3-dev
 ```
 
 Then, you can install Credential Digger module using `pip`.
@@ -124,9 +118,17 @@ credentialdigger scan https://github.com/user/repo --sqlite /path/to/data.db --s
 
 ## Docker container
 
-To have a ready-to-use instance of Credential Digger, with a user interface, you can build the docker container. 
+To have a ready-to-use instance of Credential Digger, with a user interface, you can use a docker container. 
 This option requires the installation of [Docker](https://docs.docker.com/engine/install/) and [Docker Compose](https://docs.docker.com/compose/install/).
 
+Credential Digger is published on [dockerhub](https://hub.docker.com/r/saposs/credentialdigger). You can pull the latest release
+
+```bash
+sudo docker pull saposs/credentialdigger
+```
+
+Or build and run containers with docker-compose
+
 ```bash
 git clone https://github.com/SAP/credential-digger.git
 cd credential-digger
@@ -157,11 +159,10 @@ git clone https://github.com/SAP/credential-digger.git
 cd credential-digger
 ```
 
-Install the requirements from `requirements.txt` file and install the library:
+Install the tool from source:
 
 ```bash
-pip install -r requirements.txt
-python setup.py install
+pip install .
 ```
 
 Then, you can add the rules and scan a repository as described above.

credentialdigger/cli/get_discoveries.py

@@ -225,7 +225,7 @@ def run(client, args):
     args: `argparse.Namespace`
         Arguments from command line parser.
     """
-    discoveries = [] 
+    discoveries = []
     try:
         discoveries = client.get_discoveries(
             repo_url=args.repo_url, file_name=args.filename, with_rules=args.with_rules)

credentialdigger/scanners/file_scanner.py

@@ -157,8 +157,7 @@ def scan_file(self, project_root, relative_path, **kwargs):
                 for row in file_to_scan:
                     rh = ResultHandler()
                     self.stream.scan(
-                        row if sys.version_info < (3, 8) else row.encode(
-                            'utf-8'),
+                        row.encode('utf-8'),
                         match_event_handler=rh.handle_results,
                         context=[row.strip(), relative_path, commit_id,
                                  line_number]

credentialdigger/scanners/git_scanner.py

@@ -406,7 +406,7 @@ def _regex_check(self, printable_diff, filename, commit_hash):
 
             rh = ResultHandler()
             self.stream.scan(
-                row if sys.version_info < (3, 8) else row.encode('utf-8'),
+                row.encode('utf-8'),
                 match_event_handler=rh.handle_results,
                 context=[row, filename, commit_hash, line_number])
             if rh.result:

requirements.txt

@@ -1,21 +1,21 @@
 Flask
 flask_jwt_extended
 GitPython
-hyperscan==0.2.0; python_version >= "3.8"
-hyperscan==0.1.5; python_version < "3.8"
+hyperscan==0.6.0; python_version != "3.9"
+hyperscan==0.5.0; python_version == "3.9"
 numpy
 pandas
 psycopg2-binary
 PyGithub
 python-dotenv
 pyyaml
-rich~=12.2
+rich~=13.7
 srsly>=2.4.0
-tensorflow==2.11.1; python_version >= "3.8"
-tensorflow~=2.4; python_version < "3.8"
-tensorflow-estimator==2.11.0; python_version >= "3.8"
-tensorflow-estimator~=2.4; python_version < "3.8"
-tensorflow-text==2.11.0; python_version >= "3.8"
-tensorflow-text~=2.4; python_version < "3.8"
+tensorflow==2.15.0; python_version > "3.8"
+tensorflow==2.13.1; python_version <= "3.8"
+tensorflow-estimator==2.15.0; python_version > "3.8"
+tensorflow-estimator==2.13.0; python_version <= "3.8"
+tensorflow-text==2.15.0; python_version > "3.8"
+tensorflow-text==2.13.0; python_version <= "3.8"
 tf-models-official
 transformers

setup.py

@@ -13,7 +13,7 @@ def requirements():
 
 setuptools.setup(
     name='credentialdigger',
-    version='4.11.2',
+    version='4.12.0',
     author='SAP SE',
     maintainer='Marco Rosa, Slim Trabelsi',
     maintainer_email='marco.rosa@sap.com, slim.trabelsi@sap.com',
@@ -28,7 +28,7 @@ def requirements():
         'Programming Language :: Python :: 3',
         'Operating System :: OS Independent',
     ],
-    python_requires='>3.5, <3.11',
+    python_requires='>3.7, <3.12',
     entry_points={'console_scripts': ['credentialdigger=credentialdigger'
                                       '.__main__:main']},
 )

tests/tests-requirements.txt

@@ -1,2 +1,2 @@
-parameterized~=0.8.1
-pytest~=7.3.0
+parameterized~=0.9.0
+pytest~=7.4.3

ui/Dockerfile

@@ -1,7 +1,7 @@
-FROM python:3.9-bullseye as builder
+FROM python:3.10-bookworm as builder
 
 # Install system packages
-RUN apt-get update && apt-get install -y libhyperscan5 libpq-dev gunicorn3
+RUN apt-get update && apt-get install -y libpq-dev gunicorn3
 RUN apt-get install -y dos2unix
 
 # Install python packages