feat: add specific filters for hfh

Author: matiasdaloia

src/scanoss/file_filters.py

@@ -46,6 +46,19 @@
     'copying.lib',
     'makefile',
 }
+
+DEFAULT_SKIPPED_FILES_HFH = {
+    'gradlew',
+    'gradlew.bat',
+    'mvnw',
+    'mvnw.cmd',
+    'gradle-wrapper.jar',
+    'maven-wrapper.jar',
+    'thumbs.db',
+    'babel.config.js',
+}
+
+
 # Folders to skip
 DEFAULT_SKIPPED_DIRS = {
     'nbproject',
@@ -66,9 +79,26 @@
     'test',
 }
 
+DEFAULT_SKIPPED_DIRS_HFH = {
+    'nbproject',
+    'nbbuild',
+    'nbdist',
+    '__pycache__',
+    'venv',
+    '_yardoc',
+    'eggs',
+    'wheels',
+    'htmlcov',
+    '__pypackages__',
+    'example',
+    'examples',
+}
+
 
 # Folder endings to skip
 DEFAULT_SKIPPED_DIR_EXT = {'.egg-info'}
+DEFAULT_SKIPPED_DIR_EXT_HFH = {'.egg-info'}
+
 # File extensions to skip
 DEFAULT_SKIPPED_EXT = {
     '.1',
@@ -243,6 +273,162 @@
     'sqlite3',
 }
 
+# TODO: For hfh add the .gitignore patterns
+DEFAULT_SKIPPED_EXT_HFH = {
+    '.1',
+    '.2',
+    '.3',
+    '.4',
+    '.5',
+    '.6',
+    '.7',
+    '.8',
+    '.9',
+    '.ac',
+    '.adoc',
+    '.am',
+    '.asciidoc',
+    '.bmp',
+    '.build',
+    '.cfg',
+    '.chm',
+    '.class',
+    '.cmake',
+    '.cnf',
+    '.conf',
+    '.config',
+    '.contributors',
+    '.copying',
+    '.crt',
+    '.csproj',
+    '.css',
+    '.csv',
+    '.dat',
+    '.data',
+    '.dtd',
+    '.dts',
+    '.iws',
+    '.c9',
+    '.c9revisions',
+    '.dtsi',
+    '.dump',
+    '.eot',
+    '.eps',
+    '.geojson',
+    '.gif',
+    '.glif',
+    '.gmo',
+    '.guess',
+    '.hex',
+    '.htm',
+    '.html',
+    '.ico',
+    '.iml',
+    '.in',
+    '.inc',
+    '.info',
+    '.ini',
+    '.ipynb',
+    '.jpeg',
+    '.jpg',
+    '.json',
+    '.jsonld',
+    '.lock',
+    '.log',
+    '.m4',
+    '.map',
+    '.md5',
+    '.meta',
+    '.mk',
+    '.mxml',
+    '.o',
+    '.otf',
+    '.out',
+    '.pbtxt',
+    '.pdf',
+    '.pem',
+    '.phtml',
+    '.plist',
+    '.png',
+    '.prefs',
+    '.properties',
+    '.pyc',
+    '.qdoc',
+    '.result',
+    '.rgb',
+    '.rst',
+    '.scss',
+    '.sha',
+    '.sha1',
+    '.sha2',
+    '.sha256',
+    '.sln',
+    '.spec',
+    '.sub',
+    '.svg',
+    '.svn-base',
+    '.tab',
+    '.template',
+    '.test',
+    '.tex',
+    '.tiff',
+    '.ttf',
+    '.txt',
+    '.utf-8',
+    '.vim',
+    '.wav',
+    '.woff',
+    '.woff2',
+    '.xht',
+    '.xhtml',
+    '.xml',
+    '.xpm',
+    '.xsd',
+    '.xul',
+    '.yaml',
+    '.yml',
+    '.wfp',
+    '.editorconfig',
+    '.dotcover',
+    '.pid',
+    '.lcov',
+    '.egg',
+    '.manifest',
+    '.cache',
+    '.coverage',
+    '.cover',
+    '.gem',
+    '.lst',
+    '.pickle',
+    '.pdb',
+    '.gml',
+    '.pot',
+    '.plt',
+    '.whml',
+    '.pom',
+    '.smtml',
+    '.min.js',
+    '.mf',
+    '.base64',
+    '.s',
+    '.diff',
+    '.patch',
+    '.rules',
+    # File endings
+    '-doc',
+    'config',
+    'news',
+    'readme',
+    'swiftdoc',
+    'texidoc',
+    'todo',
+    'version',
+    'ignore',
+    'manifest',
+    'sqlite',
+    'sqlite3',
+}
+
 
 class FileFilters(ScanossBase):
     """
@@ -267,6 +453,7 @@ def __init__(self, debug: bool = False, trace: bool = False, quiet: bool = False
                 skip_size (int): Size to skip
                 skip_extensions (list): Extensions to skip
                 skip_folders (list): Folders to skip
+                is_folder_hashing_scan (bool): Whether the operation is a folder hashing scan
         """
         super().__init__(debug, trace, quiet)
 
@@ -277,6 +464,7 @@ def __init__(self, debug: bool = False, trace: bool = False, quiet: bool = False
         self.skip_folders = kwargs.get('skip_folders', [])
         self.skip_size = kwargs.get('skip_size', 0)
         self.skip_extensions = kwargs.get('skip_extensions', [])
+        self.is_folder_hashing_scan = kwargs.get('is_folder_hashing_scan', False)
         self.file_folder_pat_spec = self._get_file_folder_pattern_spec(kwargs.get('operation_type', 'scanning'))
         self.size_pat_rules = self._get_size_limit_pattern_rules(kwargs.get('operation_type', 'scanning'))
 
@@ -336,36 +524,36 @@ def get_filtered_files_from_files(self, files: List[str], scan_root: str = None)
         """
         filtered_files = []
         for file_path in files:
-            if not os.path.exists(file_path) or not os.path.isfile(file_path) or os.path.islink(file_path):
-                self.print_debug(
-                    f'WARNING: File {file_path} does not exist, is not a file, or is a symbolic link. Ignoring.'
-                )
-                continue
-
             path_obj = Path(file_path)
-            if not self.hidden_files_folders and any(part.startswith('.') for part in path_obj.parts):
-                self.print_debug(f'Skipping file: {file_path} (in hidden directory or is hidden file)')
-                continue
-
             try:
                 if scan_root:
-                    rel_path = os.path.relpath(file_path, scan_root)
+                    rel_path = path_obj.relative_to(scan_root)
                 else:
-                    rel_path = os.path.relpath(file_path)
+                    rel_path = str(path_obj)
             except ValueError:
-                # If file_path is broken, symlink ignore it
                 self.print_debug(f'Ignoring file: {file_path} (broken symlink)')
                 continue
+
+            if not path_obj.exists() or not path_obj.is_file() or path_obj.is_symlink():
+                self.print_debug(
+                    f'WARNING: File {rel_path} does not exist, is not a file, or is a symbolic link. Ignoring.'
+                )
+                continue
+
+            if not self.hidden_files_folders and any(part.startswith('.') for part in path_obj.parts):
+                self.print_debug(f'Skipping file: {rel_path} (in hidden directory or is hidden file)')
+                continue
+
             if self._should_skip_file(rel_path):
                 continue
             try:
-                file_size = os.path.getsize(file_path)
+                file_size = path_obj.stat().st_size
                 if file_size == 0:
                     self.print_debug(f'Skipping file: {rel_path} (empty file)')
                     continue
                 min_size, max_size = self._get_operation_size_limits(file_path)
                 if min_size <= file_size <= max_size:
-                    filtered_files.append(rel_path)
+                    filtered_files.append(str(rel_path))
                 else:
                     self.print_debug(
                         f'Skipping file: {rel_path} (size {file_size} outside limits {min_size}-{max_size})'
@@ -379,8 +567,11 @@ def _get_file_folder_pattern_spec(self, operation_type: str = 'scanning'):
         """
         Get file path pattern specification.
 
-        :param operation_type: which operation is being performed
-        :return:  List of file path patterns
+        Args:
+            operation_type (str): Type of operation ('scanning' or 'fingerprinting')
+
+        Returns:
+            GitIgnoreSpec: GitIgnoreSpec object containing the file path patterns
         """
         patterns = self._get_operation_patterns(operation_type)
         if patterns:
@@ -391,8 +582,11 @@ def _get_size_limit_pattern_rules(self, operation_type: str = 'scanning'):
         """
         Get size limit pattern rules.
 
-        :param operation_type: which operation is being performed
-        :return: List of size limit pattern rules
+        Args:
+            operation_type (str): Type of operation ('scanning' or 'fingerprinting')
+
+        Returns:
+            List of size limit pattern rules
         """
         if self.scanoss_settings:
             size_rules = self.scanoss_settings.get_skip_sizes(operation_type)
@@ -417,6 +611,14 @@ def _get_operation_patterns(self, operation_type: str) -> List[str]:
             List[str]: Combined list of patterns to skip
         """
         patterns = []
+
+        # Default patterns for skipping directories
+        if not self.all_folders:
+            DEFAULT_SKIPPED_DIR_LIST = DEFAULT_SKIPPED_DIRS_HFH if self.is_folder_hashing_scan else DEFAULT_SKIPPED_DIRS
+            for dir_name in DEFAULT_SKIPPED_DIR_LIST:
+                patterns.append(f'{dir_name}/')
+
+        # Custom patterns added in SCANOSS settings file
         if self.scanoss_settings:
             patterns.extend(self.scanoss_settings.get_skip_patterns(operation_type))
         return patterns
@@ -505,18 +707,21 @@ def _should_skip_file(self, file_rel_path: str) -> bool:  # noqa: PLR0911
         """
         file_name = os.path.basename(file_rel_path)
 
+        DEFAULT_SKIPPED_FILES_LIST = DEFAULT_SKIPPED_FILES_HFH if self.is_folder_hashing_scan else DEFAULT_SKIPPED_FILES
+        DEFAULT_SKIPPED_EXT_LIST = DEFAULT_SKIPPED_EXT_HFH if self.is_folder_hashing_scan else DEFAULT_SKIPPED_EXT
+
         if not self.hidden_files_folders and file_name.startswith('.'):
             self.print_debug(f'Skipping file: {file_rel_path} (hidden file)')
             return True
         if self.all_extensions:
             return False
         file_name_lower = file_name.lower()
         # Look for exact files
-        if file_name_lower in DEFAULT_SKIPPED_FILES:
+        if file_name_lower in DEFAULT_SKIPPED_FILES_LIST:
             self.print_debug(f'Skipping file: {file_rel_path} (matches default skip file)')
             return True
         # Look for file endings
-        for ending in DEFAULT_SKIPPED_EXT:
+        for ending in DEFAULT_SKIPPED_EXT_LIST:
             if file_name_lower.endswith(ending):
                 self.print_debug(f'Skipping file: {file_rel_path} (matches default skip ending: {ending})')
                 return True
@@ -531,39 +736,3 @@ def _should_skip_file(self, file_rel_path: str) -> bool:  # noqa: PLR0911
             self.print_debug(f'Skipping file: {file_rel_path} (matches custom pattern)')
             return True
         return False
-
-    def _should_skip_file_for_hfh(self, file_path: Path) -> bool:
-        """
-        Check if a file should be skipped during folder hashing scan.
-
-        Args:
-            file_path (Path): The path to the file to check.
-
-        Returns:
-            bool: True if the file should be skipped, False otherwise.
-        """
-        try:
-            if (
-                any(part.startswith('.') for part in file_path.parts)  # Hidden files/folders
-                or file_path.is_symlink()  # Symlinks
-                or file_path.stat().st_size == 0  # Empty files
-            ):
-                self.print_debug(f'Skipping file: {file_path} (hidden/symlink/empty)')
-                return True
-
-            # Files ending with null
-            if file_path.suffix.lower() == '.txt':
-                try:
-                    with open(file_path, 'rb') as f:
-                        if f.read().endswith(b'\x00'):
-                            self.print_debug(f'Skipping file: {file_path} (text file ending with null)')
-                            return True
-                except (OSError, IOError):
-                    self.print_debug(f'Skipping file: {file_path} (cannot read file content)')
-                    return True
-
-            return False
-
-        except Exception as e:
-            self.print_debug(f'Error checking file {file_path}: {str(e)}')
-            return True