Add config option to ingore rules per file

saltlint/cli.py

@@ -6,33 +6,13 @@
 
 import optparse
 import sys
-import yaml
-import os
 
-# Import Salt libs
-from salt.ext import six
-
-import saltlint.linter
-import saltlint.formatters as formatters
+from saltlint import formatters, NAME, VERSION
+from saltlint.config import SaltLintConfig, SaltLintConfigError, default_rulesdir
 from saltlint.linter import RulesCollection, Runner
-from saltlint import NAME, VERSION
-
-
-def load_config(config_file):
-    config_path = config_file if config_file else ".salt-lint"
-
-    if os.path.exists(config_path):
-        with open(config_path, "r") as stream:
-            try:
-                return yaml.safe_load(stream)
-            except yaml.YAMLERROR:
-                pass
-
-    return None
 
 
-def run():
-
+def run(args=None):
     formatter = formatters.Formatter()
 
     parser = optparse.OptionParser("%prog [options] init.sls [state ...]",
@@ -45,14 +25,14 @@ def run():
                       help="specify one or more rules directories using "
                            "one or more -r arguments. Any -r flags override "
                            "the default rules in %s, unless -R is also used."
-                           % saltlint.linter.default_rulesdir)
+                           % default_rulesdir)
     parser.add_option('-R', action='store_true',
                       default=False,
                       dest='use_default_rules',
                       help="Use default rules in %s in addition to any extra "
                            "rules directories specified with -r. There is "
                            "no need to specify this if no -r flags are used."
-                           % saltlint.linter.default_rulesdir)
+                           % default_rulesdir)
     parser.add_option('-t', dest='tags',
                       action='append',
                       default=[],
@@ -77,74 +57,51 @@ def run():
                            ' is repeatable.',
                       default=[])
     parser.add_option('-c', help='Specify configuration file to use.  Defaults to ".salt-lint"')
-    options, args = parser.parse_args(sys.argv[1:])
-
-    config = load_config(options.c)
-
-    if config:
-        # TODO parse configuration options
-
-        if 'verbosity' in config:
-            options.verbosity = options.verbosity + config['verbosity']
-
-        if 'exclude_paths' in config:
-            options.exclude_paths = options.exclude_paths + config['exclude_paths']
-
-        if 'skip_list' in config:
-            options.skip_list = options.skip_list + config['skip_list']
-
-        if 'tags' in config:
-            options.tags = options.tags + config['tags']
-
-        if 'use_default_rules' in config:
-            options.use_default_rules = options.use_default_rules or config['use_default_rules']
-
-        if 'rulesdir' in config:
-            options.rulesdir = options.rulesdir + config['rulesdir']
+    (options, parsed_args) = parser.parse_args(args if args is not None else sys.argv[1:])
+
+    # Read, parse and validate the configration
+    options_dict = vars(options)
+    try:
+        config = SaltLintConfig(options_dict)
+    except SaltLintConfigError as exc:
+        print(exc)
+        return 2
 
-    if len(args) == 0 and not (options.listrules or options.listtags):
+    # Show a help message on the screen
+    if len(parsed_args) == 0 and not (options.listrules or options.listtags):
         parser.print_help(file=sys.stderr)
         return 1
 
-    if options.use_default_rules:
-        rulesdirs = options.rulesdir + [saltlint.linter.default_rulesdir]
-    else:
-        rulesdirs = options.rulesdir or [saltlint.linter.default_rulesdir]
-
-    rules = RulesCollection()
-    for rulesdir in rulesdirs:
-        rules.extend(RulesCollection.create_from_directory(rulesdir))
+    # Collect the rules from the configution
+    rules = RulesCollection(config)
+    for rulesdir in config.rulesdirs:
+        rules.extend(RulesCollection.create_from_directory(rulesdir, config))
 
+    # Show the rules listing
     if options.listrules:
         print(rules)
         return 0
 
+    # Show the tags listing
     if options.listtags:
         print(rules.listtags())
         return 0
 
-    if isinstance(options.tags, six.string_types):
-        options.tags = options.tags.split(',')
-
-    skip = set()
-    for s in options.skip_list:
-        skip.update(str(s).split(','))
-    options.skip_list = frozenset(skip)
-
-    states = set(args)
+    states = set(parsed_args)
     matches = list()
     checked_files = set()
     for state in states:
-        runner = Runner(rules, state, options.tags,
-                        options.skip_list, options.exclude_paths,
-                        options.verbosity, checked_files)
+        runner = Runner(rules, state, config, checked_files)
         matches.extend(runner.run())
 
+    # Sort the matches
     matches.sort(key=lambda x: (x.filename, x.linenumber, x.rule.id))
 
+    # Show the matches on the screen
     for match in matches:
-        print(formatter.format(match, options.colored))
+        print(formatter.format(match, config.colored))
 
+    # Return the exit code
     if len(matches):
         return 2
     else:

saltlint/config.py

@@ -0,0 +1,122 @@
+# -*- coding: utf-8 -*-
+# Copyright (c) 2019 Roald Nefs
+
+import yaml
+import os
+import sys
+import pathspec
+
+# Import Salt libs
+from salt.ext import six
+
+import saltlint.utils
+
+
+default_rulesdir = os.path.join(os.path.dirname(saltlint.utils.__file__), 'rules')
+
+
+class SaltLintConfigError(Exception):
+    pass
+
+
+class SaltLintConfig(object):
+
+    def __init__(self, options=dict()):
+        self._options = options
+        # Configuration file to use, defaults to ".salt-lint".
+        config = options.get('c')
+        file = config if config is not None else '.salt-lint'
+
+        # Read the file contents
+        if os.path.exists(file):
+            with open(file, 'r') as f:
+                content = f.read()
+        else:
+            content = None
+
+        # Parse the content of the file as YAML
+        self._parse(content)
+
+    def _parse(self, content):
+        config = dict()
+
+        # Parse the YAML content
+        if content:
+            try:
+                config = yaml.safe_load(content)
+            except Exception as exc:
+                raise SaltLintConfigError("invalid config: {}".format(exc))
+
+        # Parse verbosity
+        self.verbosity = self._options.get('verbosity', 0)
+        if 'verbosity' in config:
+            self.verbosity += config['verbosity']
+
+        # Parse exclude paths
+        self.exclude_paths = self._options.get('exclude_paths', [])
+        if 'exclude_paths' in config:
+            self.exclude_paths += config['exclude_paths']
+
+        # Parse skip list
+        skip_list = self._options.get('skip_list', [])
+        if 'skip_list' in config:
+            skip_list += config['skip_list']
+        skip = set()
+        for s in skip_list:
+            skip.update(str(s).split(','))
+        self.skip_list = frozenset(skip)
+
+        # Parse tags
+        self.tags = self._options.get('tags', [])
+        if 'tags' in config:
+            self.tags += config['tags']
+        if isinstance(self.tags, six.string_types):
+            self.tags = self.tags.split(',')
+
+        # Parse use default rules
+        use_default_rules = self._options.get('use_default_rules', False)
+        if 'use_default_rules' in config:
+            use_default_rules = use_default_rules or config['use_default_rules']
+
+        # Parse rulesdir
+        rulesdir = self._options.get('rulesdir', [])
+        if 'rulesdir' in config:
+            rulesdir += config['rulesdir']
+
+        # Determine the rules directories
+        if use_default_rules:
+            self.rulesdirs = rulesdir + [default_rulesdir]
+        else:
+            self.rulesdirs = rulesdir or [default_rulesdir]
+
+        # Parse colored
+        self.colored = self._options.get(
+            'colored',
+            hasattr(sys.stdout, 'isatty') and sys.stdout.isatty()
+        )
+
+        # Parse rule specific configuration, the configration can be listed by
+        # the rule ID and/or tag.
+        self.rules = dict()
+        if 'rules' in config and isinstance(config['rules'], dict):
+            # Read rule specific configuration from the config dict.
+            for name, rule in six.iteritems(config['rules']):
+                # Store the keys as strings.
+                self.rules[str(name)] = dict()
+
+                if 'ignore' not in rule:
+                    continue
+
+                if not isinstance(rule['ignore'], six.string_types):
+                    raise SaltLintConfigError(
+                        'invalid config: ignore should contain file patterns')
+
+                # Retrieve the pathspec.
+                self.rules[str(name)]['ignore'] = pathspec.PathSpec.from_lines(
+                    'gitwildmatch', rule['ignore'].splitlines())
+
+    def is_file_ignored(self, filepath, rule):
+        rule = str(rule)
+        if rule not in self.rules or 'ignore' not in self.rules[rule]:
+            return False
+        return self.rules[rule]['ignore'].match_file(filepath)

saltlint/linter.py

@@ -14,13 +14,14 @@
 from salt.ext import six
 
 import saltlint.utils
-
-
-default_rulesdir = os.path.join(os.path.dirname(saltlint.utils.__file__), 'rules')
+from saltlint.config import SaltLintConfig
 
 
 class SaltLintRule(object):
 
+    def __init__(self, config=None):
+        self.config = config
+
     def __repr__(self):
         return self.id + ": " + self.shortdesc
 
@@ -75,8 +76,9 @@ def matchfulltext(self, file, text):
 
 class RulesCollection(object):
 
-    def __init__(self):
+    def __init__(self, config=SaltLintConfig()):
         self.rules = []
+        self.config = config
 
     def register(self, obj):
         self.rules.append(obj)
@@ -104,10 +106,18 @@ def run(self, statefile, tags=set(), skip_list=frozenset()):
             return matches
 
         for rule in self.rules:
+            skip = False
             if not tags or not set(rule.tags).union([rule.id]).isdisjoint(tags):
                 rule_definition = set(rule.tags)
                 rule_definition.add(rule.id)
-                if set(rule_definition).isdisjoint(skip_list):
+
+                # Check if the the file is in the rule specific ignore list.
+                for definition in rule_definition:
+                    if self.config.is_file_ignored(statefile['path'], definition):
+                        skip = True
+                        break
+
+                if not skip and set(rule_definition).isdisjoint(skip_list):
                     matches.extend(rule.matchlines(statefile, text))
                     matches.extend(rule.matchfulltext(statefile, text))
 
@@ -128,9 +138,9 @@ def listtags(self):
         return "\n".join(results)
 
     @classmethod
-    def create_from_directory(cls, rulesdir):
-        result = cls()
-        result.rules = saltlint.utils.load_plugins(os.path.expanduser(rulesdir))
+    def create_from_directory(cls, rulesdir, config):
+        result = cls(config)
+        result.rules = saltlint.utils.load_plugins(os.path.expanduser(rulesdir), config)
         return result
 
 
@@ -151,19 +161,24 @@ def __repr__(self):
 
 class Runner(object):
 
-    def __init__(self, rules, state, tags, skip_list, exclude_paths,
-                 verbosity=0, checked_files=None):
+    def __init__(self, rules, state, config, checked_files=None):
         self.rules = rules
+
         self.states = set()
-        # assume state is directory
+        # Assume state is directory
         if os.path.isdir(state):
             self.states.add((os.path.join(state, 'init.sls'), 'state'))
         else:
             self.states.add((state, 'state'))
-        self.tags = tags
-        self.skip_list = skip_list
-        self._update_exclude_paths(exclude_paths)
-        self.verbosity = verbosity
+
+        # Get configuration options
+        self.config = config
+        self.tags = config.tags
+        self.skip_list = config.skip_list
+        self.verbosity = config.verbosity
+        self._update_exclude_paths(config.exclude_paths)
+
+        # Set the checked files
         if checked_files is None:
             checked_files = set()
         self.checked_files = checked_files
@@ -194,9 +209,6 @@ def run(self):
         # remove duplicates from files list
         files = [value for n, value in enumerate(files) if value not in files[:n]]
 
-        # remove duplicates from files list
-        files = [value for n, value in enumerate(files) if value not in files[:n]]
-
         # remove files that have already been checked
         files = [x for x in files if x['path'] not in self.checked_files]
         for file in files:

saltlint/utils.py

@@ -7,7 +7,7 @@
 import os
 
 
-def load_plugins(directory):
+def load_plugins(directory, config):
     result = []
     fh = None
 
@@ -17,7 +17,7 @@ def load_plugins(directory):
         try:
             fh, filename, desc = imp.find_module(pluginname, [directory])
             mod = imp.load_module(pluginname, fh, filename, desc)
-            obj = getattr(mod, pluginname)()
+            obj = getattr(mod, pluginname)(config)
             result.append(obj)
         finally:
             if fh: