feat: apply same filters from go minr

feat: create standalone presenters that implement abstractpresenter

feat: add progress tracking while scanning, clean folder-scan command arguments

feat: use progress instead of rich

feat: add folder hash sub-command, fix hfh spinner

feat: update docs

Author: matiasdaloia

CHANGELOG.md

@@ -12,6 +12,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [1.21.0] - 2025-02-10
 ### Added
 - Add folder-scan subcommand
+- Add folder-hash subcommand
 - Add AbstractPresenter class for presenting output in a given format
 - Add several reusable helper functions for constructing config objects from CLI args
 

docs/source/index.rst

@@ -228,6 +228,79 @@ Convert file format to plain, SPDX-Lite, CycloneDX or csv.
    * - --format <format>, -f <format>
      - Indicates the result output format: {plain, cyclonedx, spdxlite, csv}. (optional - default plain)
 
+--------------------------------
+Folder Scanning: folder-scan, fs
+--------------------------------
+
+Performs a comprehensive scan of a directory using folder hashing to identify components and their matches.
+
+.. code-block:: bash
+
+   scanoss-py folder-scan <directory>
+
+.. list-table:: 
+   :widths: 20 30
+   :header-rows: 1
+
+   * - Argument
+     - Description
+   * - --output <file name>, -o <file name>
+     - Output result file name (optional - default STDOUT)
+   * - --format <format>, -f <format>
+     - Output format: {json} (optional - default json)
+   * - --timeout <seconds>, -M <seconds>
+     - Timeout in seconds for API communication (optional - default 600)
+   * - --best-match, -bm
+     - Enable best match mode (optional - default: False)
+   * - --threshold <1-100>
+     - Threshold for result matching (optional - default: 100)
+   * - --settings <file>, -st <file>
+     - Settings file to use for scanning (optional - default scanoss.json)
+   * - --skip-settings-file, -stf
+     - Skip default settings file (scanoss.json) if it exists
+   * - --key <token>, -k <token>
+     - SCANOSS API Key token (optional - not required for default OSSKB URL)
+   * - --proxy <url>
+     - Proxy URL to use for connections
+   * - --pac <file/url>
+     - Proxy auto configuration. Specify a file, http url or "auto"
+   * - --ca-cert <file>
+     - Alternative certificate PEM file
+   * - --api2url <url>
+     - SCANOSS gRPC API 2.0 URL (optional - default: https://api.osskb.org)
+   * - --grpc-proxy <url>
+     - GRPC Proxy URL to use for connections
+
+--------------------------------
+Folder Hashing: folder-hash, fh
+--------------------------------
+
+Generates cryptographic hashes for files in a given directory and its subdirectories.
+
+.. code-block:: bash
+
+   scanoss-py folder-hash <directory>
+
+.. list-table:: 
+   :widths: 20 30
+   :header-rows: 1
+
+   * - Argument
+     - Description
+   * - --output <file name>, -o <file name>
+     - Output result file name (optional - default STDOUT)
+   * - --format <format>, -f <format>
+     - Output format: {json} (optional - default json)
+   * - --settings <file>, -st <file>
+     - Settings file to use for scanning (optional - default scanoss.json)
+   * - --skip-settings-file, -stf
+     - Skip default settings file (scanoss.json) if it exists
+
+Both commands also support these general options:
+   * --debug, -d: Enable debug messages
+   * --trace, -t: Enable trace messages
+   * --quiet, -q: Enable quiet mode
+
 -----------------
 Component:
 -----------------

setup.cfg

@@ -38,6 +38,7 @@ install_requires =
     packageurl-python
     pathspec
     jsonschema
+    crc
 
 
 [options.extras_require]

src/scanoss/cli.py

@@ -30,6 +30,10 @@
 
 import pypac
 
+from scanoss.scanners.folder_hasher import (
+    FolderHasher,
+    create_folder_hasher_config_from_args,
+)
 from scanoss.scanossgrpc import (
     ScanossGrpc,
     ScanossGrpcError,
@@ -316,6 +320,7 @@ def setup_args() -> None:  # noqa: PLR0915
     for p in [c_crypto, c_vulns, c_semgrep]:
         p.add_argument('--purl', '-p', type=str, nargs='*', help='Package URL - PURL to process.')
         p.add_argument('--input', '-i', type=str, help='Input file name')
+
     # Common Component sub-command options
     for p in [c_crypto, c_vulns, c_search, c_versions, c_semgrep, c_provenance]:
         p.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).')
@@ -474,35 +479,68 @@ def setup_args() -> None:  # noqa: PLR0915
     )
     p_undeclared.set_defaults(func=inspect_undeclared)
 
+    # Sub-command: folder-scan
     p_folder_scan = subparsers.add_parser(
         'folder-scan',
+        aliases=['fs'],
         description=f'Scan the given directory using folder hashing: {__version__}',
         help='Scan the given directory using folder hashing',
     )
     p_folder_scan.add_argument('scan_dir', metavar='FILE/DIR', type=str, nargs='?', help='The root directory to scan')
     p_folder_scan.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).')
+    p_folder_scan.add_argument(
+        '--timeout',
+        '-M',
+        type=int,
+        default=600,
+        help='Timeout (in seconds) for API communication (optional - default 600)',
+    )
     p_folder_scan.add_argument(
         '--format',
         '-f',
         type=str,
-        choices=['plain', 'json'],
-        help='Result output format (optional - default: plain)',
+        choices=['json'],
+        default='json',
+        help='Result output format (optional - default: json)',
     )
     p_folder_scan.add_argument(
         '--best-match',
         '-bm',
         action='store_true',
+        default=False,
         help='Enable best match mode (optional - default: False)',
     )
     p_folder_scan.add_argument(
         '--threshold',
         type=int,
+        choices=range(1, 101),
+        metavar='1-100',
+        default=100,
         help='Threshold for result matching (optional - default: 100)',
     )
     p_folder_scan.set_defaults(func=folder_hashing_scan)
 
+    # Sub-command: folder-hash
+    p_folder_hash = subparsers.add_parser(
+        'folder-hash',
+        aliases=['fh'],
+        description=f'Produce a folder hash for the given directory: {__version__}',
+        help='Produce a folder hash for the given directory',
+    )
+    p_folder_hash.add_argument('scan_dir', metavar='FILE/DIR', type=str, nargs='?', help='A file or folder to scan')
+    p_folder_hash.add_argument('--output', '-o', type=str, help='Output result file name (optional - default stdout).')
+    p_folder_hash.add_argument(
+        '--format',
+        '-f',
+        type=str,
+        choices=['json'],
+        default='json',
+        help='Result output format (optional - default: json)',
+    )
+    p_folder_hash.set_defaults(func=folder_hash)
+
     # Scanoss settings options
-    for p in [p_folder_scan, p_scan, p_wfp]:
+    for p in [p_folder_scan, p_scan, p_wfp, p_folder_hash]:
         p.add_argument(
             '--settings',
             '-st',
@@ -530,7 +568,7 @@ def setup_args() -> None:  # noqa: PLR0915
         p.add_argument('-s', '--status', type=str, help='Save summary data into Markdown file')
 
     # Global Scan command options
-    for p in [p_scan, p_folder_scan]:
+    for p in [p_scan]:
         p.add_argument(
             '--apiurl', type=str, help='SCANOSS API URL (optional - default: https://api.osskb.org/scan/direct)'
         )
@@ -614,6 +652,7 @@ def setup_args() -> None:  # noqa: PLR0915
         p_undeclared,
         p_copyleft,
         p_folder_scan,
+        p_folder_hash,
     ]:
         p.add_argument('--debug', '-d', action='store_true', help='Enable debug messages')
         p.add_argument('--trace', '-t', action='store_true', help='Enable trace messages, including API posts')
@@ -1499,6 +1538,39 @@ def folder_hashing_scan(parser, args):
         sys.exit(1)
 
 
+def folder_hash(parser, args):
+    """Run the "folder-hash" sub-command
+
+    Args:
+        parser (ArgumentParser): command line parser object
+        args (Namespace): Parsed arguments
+    """
+    try:
+        if not args.scan_dir:
+            print_stderr('ERROR: Please specify a directory to scan')
+            parser.parse_args([args.subparser, '-h'])
+            sys.exit(1)
+
+        if not os.path.exists(args.scan_dir) or not os.path.isdir(args.scan_dir):
+            print_stderr(f'ERROR: The specified directory {args.scan_dir} does not exist')
+            sys.exit(1)
+
+        folder_hasher_config = create_folder_hasher_config_from_args(args)
+        scanoss_settings = get_scanoss_settings_from_args(args)
+
+        folder_hasher = FolderHasher(
+            scan_dir=args.scan_dir,
+            config=folder_hasher_config,
+            scanoss_settings=scanoss_settings,
+        )
+
+        folder_hasher.hash_directory(args.scan_dir)
+        folder_hasher.present(output_file=args.output, output_format=args.format)
+    except Exception as e:
+        print_stderr(f'ERROR: {e}')
+        sys.exit(1)
+
+
 def get_scanoss_settings_from_args(args):
     scanoss_settings = None
     if not args.skip_settings_file:

src/scanoss/file_filters.py

@@ -514,3 +514,39 @@ def _should_skip_file(self, file_rel_path: str) -> bool:  # noqa: PLR0911
             self.print_debug(f'Skipping file: {file_rel_path} (matches custom pattern)')
             return True
         return False
+
+    def _should_skip_file_for_hfh(self, file_path: Path) -> bool:
+        """
+        Check if a file should be skipped during folder hashing scan.
+
+        Args:
+            file_path (Path): The path to the file to check.
+
+        Returns:
+            bool: True if the file should be skipped, False otherwise.
+        """
+        try:
+            if (
+                any(part.startswith('.') for part in file_path.parts)  # Hidden files/folders
+                or file_path.is_symlink()  # Symlinks
+                or file_path.stat().st_size == 0  # Empty files
+            ):
+                self.print_debug(f'Skipping file: {file_path} (hidden/symlink/empty)')
+                return True
+
+            # Files ending with null
+            if file_path.suffix.lower() == '.txt':
+                try:
+                    with open(file_path, 'rb') as f:
+                        if f.read().endswith(b'\x00'):
+                            self.print_debug(f'Skipping file: {file_path} (text file ending with null)')
+                            return True
+                except (OSError, IOError):
+                    self.print_debug(f'Skipping file: {file_path} (cannot read file content)')
+                    return True
+
+            return False
+
+        except Exception as e:
+            self.print_debug(f'Error checking file {file_path}: {str(e)}')
+            return True