By default auto discover files of any type, and run any applicable to…

…ols on them. (#432) (#435)

CHANGELOG.md

@@ -8,6 +8,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 
 ### Added
 
+### Changed
+
+- Default behavior for Statick will now run all available discovery plugins, and run all tool plugins where
+  their desired source files are available, then output results only on the terminal. (#432, #435)
+  The old default behavior was to run the "sei_cert" profile, this is still doable via either of the
+  following arguments: `--profile sei_cert.yaml` or `--level sei_cert`
+
 ### Fixed
 
 ### Removed

README.md

@@ -17,7 +17,7 @@ Best practices recommend running multiple tools to get the best results.
 Statick has plugins that interface with a large number of static analysis and linting tools,
 allowing you to run a single tool to get all the results at once.
 
-Many tools are known for generating a large number of fasle positives so Statick provides multiple ways to add
+Many tools are known for generating a large number of false positives so Statick provides multiple ways to add
 exceptions to suppress false positives.
 The types of warnings to generate is highly dependent on your specific project, and Statick makes it easy to run each
 tool with custom flags to tailor the tools for the issues you care about.
@@ -260,7 +260,11 @@ The `default` key lists the _level_ to run if no specific _level_ listed for a p
 
 The `packages` key lists _packages_ and override levels to run for those packages.
 
-With the built-in configuration files the default _profile_ uses `sei_cert` as the default _level_.
+With the built-in configuration files the default _profile_ uses `default` as the default _level_.
+This _level_ runs all available _discovery_ plugins, sets all available _tools_ to use their default flags,
+and only runs the _print_to_console_ reporting plugin (which outputs results to the terminal).
+
+With the built-in configuration files another useful _profile_ uses the `sei_cert` _level_.
 This _level_ sets all available _tools_ to use flags that find issues listed in
 Carnegie Mellon University Software Engineering Institute
 "CERT C++ Coding Standard: Rules for Developing Safe, Reliable, and Secure Systems".
@@ -354,7 +358,7 @@ File Type | Extensions
 :-------- | :---------
 catkin    | `CMakeLists.txt` and `package.xml`
 C         | `.c`, `.cc`, `.cpp`, `.cxx`, `.h`, `.hxx`, `.hpp`
-CMake     | `CMakeLists.txt`
+CMake     | `CMakeLists.txt`, `.cmake`
 groovy    | `.groovy`, `.gradle`, `Jenkinsfile*`
 java      | `.class`, `.java`
 Maven     | `pom.xml`

statick_tool/config.py

@@ -15,8 +15,14 @@ class Config:
     Sets what flags are used for each plugin at those levels.
     """
 
-    def __init__(self, base_file: Optional[str], user_file: Optional[str] = "") -> None:
+    def __init__(
+        self,
+        base_file: Optional[str],
+        user_file: Optional[str] = "",
+        default_level: Optional[str] = "default",
+    ) -> None:
         """Initialize configuration."""
+        self.default_level = default_level
         if base_file is None or not os.path.exists(base_file):
             self.config: Any = []
             return
@@ -67,6 +73,10 @@ def has_level(self, level: Optional[str]) -> bool:
     def get_enabled_plugins(self, level: str, plugin_type: str) -> List[str]:
         """Get what plugins are enabled for a certain level."""
         plugins: List[str] = []
+
+        if level == self.default_level:
+            return plugins
+
         for level_type in self.config["levels"][level]:
             if (
                 plugin_type in level_type

statick_tool/plugins/tool/do_nothing_tool_plugin.py

@@ -0,0 +1,30 @@
+"""Do nothing, this is primarily useful for testing purposes."""
+from typing import List, Optional
+
+from statick_tool.issue import Issue
+from statick_tool.package import Package
+from statick_tool.tool_plugin import ToolPlugin
+
+
+class DoNothingToolPlugin(ToolPlugin):
+    """Do nothing, this is primarily useful for testing purposes."""
+
+    def get_name(self) -> str:
+        """Get name of tool."""
+        return "do_nothing"
+
+    def get_file_types(self) -> List[str]:
+        """Return a list of file types the plugin can scan."""
+        return []
+
+    def process_files(
+        self, package: Package, level: str, files: List[str], user_flags: List[str]
+    ) -> Optional[List[str]]:
+        """Run tool and gather output."""
+        return []
+
+    def parse_output(
+        self, total_output: List[str], package: Optional[Package] = None
+    ) -> List[Issue]:
+        """Parse tool output and report issues."""
+        return []

statick_tool/plugins/tool/do_nothing_tool_plugin.yapsy-plugin

@@ -0,0 +1,3 @@
+[Core]
+Name = Do Nothing Tool Plugin
+Module = do_nothing_tool_plugin

statick_tool/rsc/config.yaml

@@ -1,7 +1,7 @@
 levels:
   discovery_only:
-    reporting:
-      print_to_console:
+    tool:
+      do_nothing:
 
   sei_cert:
     reporting:

statick_tool/rsc/profile.yaml

@@ -1,4 +1,4 @@
-default: sei_cert
+default: default
 
 packages:
 

statick_tool/statick.py

@@ -24,11 +24,12 @@
 from statick_tool.tool_plugin import ToolPlugin
 
 
-class Statick:
+class Statick:  # pylint: disable=too-many-instance-attributes
     """Code analysis front-end."""
 
     def __init__(self, user_paths: List[str]) -> None:
         """Initialize Statick."""
+        self.default_level = "default"
         self.resources = Resources(user_paths)
 
         self.manager = PluginManager()
@@ -89,6 +90,7 @@ def get_config(self, args: argparse.Namespace) -> None:
             self.config = Config(
                 self.resources.get_file(base_config_filename),
                 self.resources.get_file(user_config_filename),
+                self.default_level,
             )
         except OSError as ex:
             logging.error(
@@ -272,7 +274,9 @@ def run(
             logging.error("Level is not valid.")
             return None, False
 
-        if not self.config or not self.config.has_level(level):
+        if not self.config or (
+            level != self.default_level and not self.config.has_level(level)
+        ):
             logging.error("Can't find specified level %s in config!", level)
             return None, False
 
@@ -327,7 +331,7 @@ def run(
 
         discovery_plugins = self.config.get_enabled_discovery_plugins(level)
         if not discovery_plugins:
-            discovery_plugins = list(self.discovery_plugins.keys())
+            discovery_plugins = list(self.discovery_plugins)
         plugins_ran: List[Any] = []
         for plugin_name in discovery_plugins:
             if plugin_name not in self.discovery_plugins:
@@ -358,6 +362,8 @@ def run(
 
         logging.info("---Tools---")
         enabled_plugins = self.config.get_enabled_tool_plugins(level)
+        if not enabled_plugins:
+            enabled_plugins = list(self.tool_plugins)
         plugins_to_run = copy.copy(enabled_plugins)
         plugins_ran = []
         plugin_dependencies: List[str] = []
@@ -422,7 +428,10 @@ def run(
         logging.info("---Reporting---")
         reporting_plugins = self.config.get_enabled_reporting_plugins(level)
         if not reporting_plugins:
-            reporting_plugins = self.reporting_plugins.keys()  # type: ignore
+            if "print_to_console" in self.reporting_plugins:
+                reporting_plugins = ["print_to_console"]
+            else:
+                reporting_plugins = list(self.reporting_plugins)
         for plugin_name in reporting_plugins:
             if plugin_name not in self.reporting_plugins:
                 logging.error("Can't find specified reporting plugin %s!", plugin_name)
@@ -553,32 +562,29 @@ def run_workspace(
                             success = False
 
         enabled_reporting_plugins: List[str] = []
-        available_reporting_plugins = {}
-        for plugin_info in self.manager.getPluginsOfCategory("Reporting"):
-            available_reporting_plugins[
-                plugin_info.plugin_object.get_name()
-            ] = plugin_info.plugin_object
 
         # Make a fake 'all' package for reporting
         dummy_all_package = Package("all_packages", parsed_args.path)
         level = self.get_level(dummy_all_package.path, parsed_args)
         if level is not None and self.config is not None:
             if not self.config or not self.config.has_level(level):
                 logging.error("Can't find specified level %s in config!", level)
-                enabled_reporting_plugins = list(available_reporting_plugins)
             else:
                 enabled_reporting_plugins = self.config.get_enabled_reporting_plugins(
                     level
                 )
 
         if not enabled_reporting_plugins:
-            enabled_reporting_plugins = list(available_reporting_plugins)
+            if "print_to_console" in self.reporting_plugins:
+                enabled_reporting_plugins = ["print_to_console"]
+            else:
+                enabled_reporting_plugins = list(self.reporting_plugins)
 
         plugin_context = PluginContext(parsed_args, self.resources, self.config)  # type: ignore
         plugin_context.args.output_directory = parsed_args.output_directory
 
         for plugin_name in enabled_reporting_plugins:
-            if plugin_name not in available_reporting_plugins:
+            if plugin_name not in self.reporting_plugins:
                 logging.error("Can't find specified reporting plugin %s!", plugin_name)
                 continue
             plugin = self.reporting_plugins[plugin_name]

tests/plugins/tool/do_nothing_tool_plugin/test_do_nothing_tool_plugin.py

@@ -0,0 +1,66 @@
+"""Unit tests for the do nothing tool plugin."""
+import os
+
+from yapsy.PluginManager import PluginManager
+
+import statick_tool
+from statick_tool.package import Package
+from statick_tool.plugins.tool.do_nothing_tool_plugin import DoNothingToolPlugin
+from statick_tool.tool_plugin import ToolPlugin
+
+
+def setup_do_nothing_tool_plugin():
+    """Create and return an instance of the do nothing plugin."""
+    plugin = DoNothingToolPlugin()
+    return plugin
+
+
+def test_do_nothing_tool_plugin_found():
+    """Test that the plugin manager finds the do nothing tool plugin."""
+    manager = PluginManager()
+    # Get the path to statick_tool/__init__.py, get the directory part, and
+    # add 'plugins' to that to get the standard plugins dir
+    manager.setPluginPlaces(
+        [os.path.join(os.path.dirname(statick_tool.__file__), "plugins")]
+    )
+    manager.setCategoriesFilter(
+        {
+            "Tool": ToolPlugin,
+        }
+    )
+    manager.collectPlugins()
+    assert any(
+        plugin_info.plugin_object.get_name() == "do_nothing"
+        for plugin_info in manager.getPluginsOfCategory("Tool")
+    )
+    assert any(
+        plugin_info.name == "Do Nothing Tool Plugin"
+        for plugin_info in manager.getPluginsOfCategory("Tool")
+    )
+
+
+def test_do_nothing_tool_plugin_get_file_types():
+    """Integration test: Make sure the do_nothing output hasn't changed."""
+    plugin = setup_do_nothing_tool_plugin()
+    assert not plugin.get_file_types()
+
+
+def test_do_nothing_tool_plugin_process_files():
+    """Integration test: Make sure the do_nothing output hasn't changed."""
+    plugin = setup_do_nothing_tool_plugin()
+    package = Package(
+        "valid_package", os.path.join(os.path.dirname(__file__), "valid_package")
+    )
+    package["python_src"] = [
+        os.path.join(os.path.dirname(__file__), "valid_package", "basic.py")
+    ]
+    output = plugin.process_files(package, "level", package["python_src"], [])
+    assert not output
+
+
+def test_do_nothing_tool_plugin_parse_output():
+    """Verify that we can parse the normal output of do_nothing."""
+    plugin = setup_do_nothing_tool_plugin()
+    output = "would reformat /home/user/valid_package/basic.py"
+    issues = plugin.parse_output([output])
+    assert not issues

tests/plugins/tool/do_nothing_tool_plugin/valid_package/basic.py

@@ -0,0 +1 @@
+import subprocess  # NOQA

tests/statick/rsc/config-discovery-dependency.yaml

@@ -2,3 +2,5 @@ levels:
   custom:
     discovery:
       - cmake
+    tool:
+      - do_nothing