Add automated tests for plugins

Test infrastructure and sample confs/rules/traces for plugins
automated tests:

New test cases are in falco_tests_plugins.yaml and cover:
- Listing plugins and fields when plugins are loaded.
- Basic cloudtrail + json plugin on a fake cloudtrail json file and a
  sample rule that uses both plugins.
- Conflicts between source/extractor plugins
- Incompatible plugin api
- Wrong plugin path
- Checking for warnings when reading rules with unnown sources (e.g. when plugins are not loaded)

Some test-only plugins written in C are in test/plugins and built on
the fly. (They aren't included in packages of course).

The test framework needed some small changes to handle these tests:
- Add a mode to not check detection counts at all (for --list/--list-plugins)
- addl_cmdline_opts to allow specifying --list/--list-plugins
- Using DOTALL when matching stderr/stdout (allows multi-line matches more easily)

Signed-off-by: Mark Stemm <mark.stemm@gmail.com>

test/CMakeLists.txt

@@ -1,4 +1,7 @@
 add_subdirectory(trace_files)
 
 add_custom_target(test-trace-files ALL)
-add_dependencies(test-trace-files trace-files-base-scap trace-files-psp trace-files-k8s-audit)
+add_dependencies(test-trace-files trace-files-base-scap trace-files-psp trace-files-k8s-audit trace-files-plugins)
+
+add_subdirectory(plugins)
+add_subdirectory(confs/plugins)

test/confs/plugins/CMakeLists.txt

@@ -0,0 +1,16 @@
+# This list is populated at cmake time, not build time
+file(GLOB test_conf_files
+	"${CMAKE_CURRENT_SOURCE_DIR}/*.yaml")
+
+foreach(conf_file_path ${test_conf_files})
+	get_filename_component(conf_file ${conf_file_path} NAME)
+	add_custom_target(test-conf-${conf_file} ALL
+		DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/${conf_file})
+	add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/${conf_file}
+		COMMAND sed -e s!BUILD_DIR!${CMAKE_BINARY_DIR}! < ${CMAKE_CURRENT_SOURCE_DIR}/${conf_file} > ${CMAKE_CURRENT_BINARY_DIR}/${conf_file}
+		DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/${conf_file})
+	list(APPEND PLUGINS_CONF_FILES_TARGETS test-conf-${conf_file})
+endforeach()
+
+add_custom_target(conf-files-plugins ALL)
+add_dependencies(conf-files-plugins ${PLUGINS_CONF_FILES_TARGETS})

test/confs/plugins/cloudtrail_json_create_instances.yaml

@@ -0,0 +1,14 @@
+stdout_output:
+  enabled: true
+
+plugins:
+  - name: cloudtrail
+    library_path: BUILD_DIR/cloudtrail-plugin-prefix/src/cloudtrail-plugin/libcloudtrail.so
+    init_config: ""
+    open_params: "BUILD_DIR/test/trace_files/plugins/alice_start_instances.json"
+  - name: json
+    library_path: BUILD_DIR/json-plugin-prefix/src/json-plugin/libjson.so
+    init_config: ""
+
+# Optional
+load_plugins: [cloudtrail, json]

test/confs/plugins/cloudtrail_json_create_instances_bigevent.yaml

@@ -0,0 +1,14 @@
+stdout_output:
+  enabled: true
+
+plugins:
+  - name: cloudtrail
+    library_path: BUILD_DIR/cloudtrail-plugin-prefix/src/cloudtrail-plugin/libcloudtrail.so
+    init_config: ""
+    open_params: "BUILD_DIR/test/trace_files/plugins/alice_start_instances_bigevent.json"
+  - name: json
+    library_path: BUILD_DIR/json-plugin-prefix/src/json-plugin/libjson.so
+    init_config: ""
+
+# Optional
+load_plugins: [cloudtrail, json]

test/confs/plugins/incompatible_extract_sources.yaml

@@ -0,0 +1,14 @@
+stdout_output:
+  enabled: true
+
+plugins:
+  - name: cloudtrail
+    library_path: BUILD_DIR/cloudtrail-plugin-prefix/src/cloudtrail-plugin/libcloudtrail.so
+    init_config: ""
+    open_params: ""
+  - name: test_extract_p1
+    library_path: BUILD_DIR/test/plugins/libtest_extract_p1.so
+    init_config: ""
+
+# Optional
+load_plugins: [cloudtrail, test_extract_p1]

test/confs/plugins/incompatible_plugin_api.yaml

@@ -0,0 +1,10 @@
+stdout_output:
+  enabled: true
+
+plugins:
+  - name: incompatible_plugin_api
+    library_path: BUILD_DIR/test/plugins/libtest_incompat_api.so
+    init_config: ""
+
+# Optional
+load_plugins: [incompatible_plugin_api]

test/confs/plugins/multiple_source_plugins.yaml

@@ -0,0 +1,15 @@
+stdout_output:
+  enabled: true
+
+plugins:
+  - name: cloudtrail
+    library_path: BUILD_DIR/cloudtrail-plugin-prefix/src/cloudtrail-plugin/libcloudtrail.so
+    init_config: ""
+    open_params: "BUILD_DIR/test/trace_files/plugins/alice_start_instances.json"
+  - name: test_source
+    library_path: BUILD_DIR/test/plugins/libtest_source.so
+    init_config: ""
+    open_params: ""
+
+# Optional
+load_plugins: [cloudtrail, test_source]

test/confs/plugins/overlap_extract_sources.yaml

@@ -0,0 +1,17 @@
+stdout_output:
+  enabled: true
+
+plugins:
+  - name: test_source
+    library_path: BUILD_DIR/test/plugins/libtest_source.so
+    init_config: ""
+    open_params: ""
+  - name: test_extract_p1
+    library_path: BUILD_DIR/test/plugins/libtest_extract_p1.so
+    init_config: ""
+  - name: test_extract_p2
+    library_path: BUILD_DIR/test/plugins/libtest_extract_p2.so
+    init_config: ""
+
+# Optional
+load_plugins: [test_source, test_extract_p1, test_extract_p2]

test/confs/plugins/wrong_plugin_path.yaml

@@ -0,0 +1,10 @@
+stdout_output:
+  enabled: true
+
+plugins:
+  - name: wrong_plugin_path
+    library_path: BUILD_DIR/test/plugins/wrong_plugin_path.so
+    init_config: ""
+
+# Optional
+load_plugins: [wrong_plugin_path]

test/falco_test.py

@@ -82,6 +82,7 @@ def setUp(self):
 
         self.exit_status = self.params.get('exit_status', '*', default=0)
         self.should_detect = self.params.get('detect', '*', default=False)
+        self.check_detection_counts = self.params.get('check_detection_counts', '*', default=True)
         self.trace_file = self.params.get('trace_file', '*', default='')
 
         if self.trace_file and not os.path.isabs(self.trace_file):
@@ -94,6 +95,7 @@ def setUp(self):
             'json_include_tags_property', '*', default=True)
         self.all_events = self.params.get('all_events', '*', default=False)
         self.priority = self.params.get('priority', '*', default='debug')
+        self.addl_cmdline_opts = self.params.get('addl_cmdline_opts', '*', default='')
         self.rules_file = self.params.get(
             'rules_file', '*', default=os.path.join(self.basedir, '../rules/falco_rules.yaml'))
 
@@ -130,6 +132,7 @@ def setUp(self):
 
         self.conf_file = self.params.get(
             'conf_file', '*', default=os.path.join(self.basedir, '../falco.yaml'))
+        self.conf_file = self.conf_file.replace("BUILD_DIR", build_dir)
         if not os.path.isabs(self.conf_file):
             self.conf_file = os.path.join(self.basedir, self.conf_file)
 
@@ -617,9 +620,9 @@ def test(self):
                 self.log.debug("Converted Rules: {}".format(psp_rules))
 
         # Run falco
-        cmd = '{} {} {} -c {} {} -o json_output={} -o json_include_output_property={} -o json_include_tags_property={} -o priority={} -v'.format(
+        cmd = '{} {} {} -c {} {} -o json_output={} -o json_include_output_property={} -o json_include_tags_property={} -o priority={} -v {}'.format(
             self.falco_binary_path, self.rules_args, self.disabled_args, self.conf_file, trace_arg, self.json_output,
-            self.json_include_output_property, self.json_include_tags_property, self.priority)
+            self.json_include_output_property, self.json_include_tags_property, self.priority, self.addl_cmdline_opts)
 
         for tag in self.disable_tags:
             cmd += ' -T {}'.format(tag)
@@ -650,13 +653,13 @@ def test(self):
                 self.fail("Stdout was not exactly {}".format(self.stderr_is))
 
         for pattern in self.stderr_contains:
-            match = re.search(pattern, res.stderr.decode("utf-8"))
+            match = re.search(pattern, res.stderr.decode("utf-8"), re.DOTALL)
             if match is None:
                 self.fail(
                     "Stderr of falco process did not contain content matching {}".format(pattern))
 
         for pattern in self.stdout_contains:
-            match = re.search(pattern, res.stdout.decode("utf-8"))
+            match = re.search(pattern, res.stdout.decode("utf-8"), re.DOTALL)
             if match is None:
                 self.fail("Stdout of falco process '{}' did not contain content matching {}".format(
                     res.stdout.decode("utf-8"), pattern))
@@ -684,7 +687,7 @@ def test(self):
         self.check_rules_warnings(res)
         if len(self.rules_events) > 0:
             self.check_rules_events(res)
-        if len(self.validate_rules_file) == 0:
+        if len(self.validate_rules_file) == 0 and self.check_detection_counts:
             self.check_detections(res)
         if len(self.detect_counts) > 0:
             self.check_detections_by_rule(res)

test/falco_tests_plugins.yaml

@@ -0,0 +1,106 @@
+#
+# Copyright (C) 2021 The Falco Authors.
+#
+# This file is part of Falco.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+trace_files: !mux
+
+  list_plugins:
+    check_detection_counts: False
+    rules_file:
+      - rules/plugins/cloudtrail_create_instances.yaml
+    conf_file: BUILD_DIR/test/confs/plugins/cloudtrail_json_create_instances.yaml
+    addl_cmdline_opts: --list-plugins
+    stdout_contains: "2 Plugins Loaded.*Name: cloudtrail.*Type: source plugin.*Name: json.*Type: extractor plugin"
+
+  list_plugin_fields:
+    check_detection_counts: False
+    rules_file:
+      - rules/plugins/cloudtrail_create_instances.yaml
+    conf_file: BUILD_DIR/test/confs/plugins/cloudtrail_json_create_instances.yaml
+    addl_cmdline_opts: --list
+    stdout_contains: "ct.id"
+
+  detect_create_instance:
+    detect: True
+    detect_level: INFO
+    rules_file:
+      - rules/plugins/cloudtrail_create_instances.yaml
+    detect_counts:
+      - 'Cloudtrail Create Instance': 1
+    conf_file: BUILD_DIR/test/confs/plugins/cloudtrail_json_create_instances.yaml
+
+  detect_create_instance_bigevent:
+    detect: True
+    detect_level: INFO
+    rules_file:
+      - rules/plugins/cloudtrail_create_instances.yaml
+    detect_counts:
+      - 'Cloudtrail Create Instance': 1
+    conf_file: BUILD_DIR/test/confs/plugins/cloudtrail_json_create_instances_bigevent.yaml
+
+  multiple_source_plugins:
+    exit_status: 1
+    stderr_contains: "Runtime error: Can not load multiple source plugins. cloudtrail already loaded. Exiting."
+    conf_file: BUILD_DIR/test/confs/plugins/multiple_source_plugins.yaml
+    rules_file:
+      - rules/plugins/cloudtrail_create_instances.yaml
+
+  incompatible_extract_sources:
+    exit_status: 1
+    stderr_contains: "Runtime error: Extractor plugin not compatible with event source aws_cloudtrail. Exiting."
+    conf_file: BUILD_DIR/test/confs/plugins/incompatible_extract_sources.yaml
+    rules_file:
+      - rules/plugins/cloudtrail_create_instances.yaml
+
+  overlap_extract_sources:
+    exit_status: 1
+    stderr_contains: "Runtime error: Extractor plugins have overlapping compatible event source test_source. Exiting."
+    conf_file: BUILD_DIR/test/confs/plugins/overlap_extract_sources.yaml
+    rules_file:
+      - rules/plugins/cloudtrail_create_instances.yaml
+
+  incompat_plugin_api:
+    exit_status: 1
+    stderr_contains: "Unsupported plugin required api version 10000000.0.0"
+    conf_file: BUILD_DIR/test/confs/plugins/incompatible_plugin_api.yaml
+    rules_file:
+      - rules/plugins/cloudtrail_create_instances.yaml
+
+  incompat_plugin_rules_version:
+    exit_status: 1
+    stderr_contains: "Runtime error: Plugin cloudtrail version 0.1.0 not compatible with required plugin version 100000.0.0. Exiting."
+    conf_file: BUILD_DIR/test/confs/plugins/cloudtrail_json_create_instances.yaml
+    rules_file:
+      - rules/plugins/cloudtrail_incompat_plugin_version.yaml
+
+  wrong_plugin_path:
+    exit_status: 1
+    stderr_contains: "error loading plugin.*No such file or directory. Exiting"
+    conf_file: BUILD_DIR/test/confs/plugins/wrong_plugin_path.yaml
+    rules_file:
+      - rules/plugins/cloudtrail_incompat_plugin_version.yaml
+
+  no_plugins_unknown_source:
+    detect: False
+    rules_file:
+      - rules/plugins/cloudtrail_create_instances.yaml
+    trace_file: trace_files/empty.scap
+    rules_warning:
+      - Cloudtrail Create Instance
+    stderr_contains: "Rule Cloudtrail Create Instance: warning .unknown-source.: unknown source aws_cloudtrail, skipping"
+
+

test/plugins/CMakeLists.txt

@@ -0,0 +1,13 @@
+add_library(test_extract_p1 SHARED test_extract.cpp)
+add_library(test_extract_p2 SHARED test_extract.cpp)
+add_library(test_source SHARED test_source.cpp)
+add_library(test_incompat_api SHARED incompat_api.cpp)
+
+target_include_directories(
+	test_extract_p1 PUBLIC "${LIBSCAP_INCLUDE_DIRS}")
+
+target_include_directories(
+	test_extract_p2 PUBLIC "${LIBSCAP_INCLUDE_DIRS}")
+
+target_include_directories(
+	test_source PUBLIC "${LIBSCAP_INCLUDE_DIRS}")

test/plugins/incompat_api.cpp

@@ -0,0 +1,28 @@
+/*
+Copyright (C) 2021 The Falco Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+#include <string.h>
+#include <stdlib.h>
+
+// Don't need any function other than plugin_get_required_api_version,
+// plugin load will fail after that.
+static const char *pl_required_api_version = "10000000.0.0";
+
+extern "C"
+const char* plugin_get_required_api_version()
+{
+	return pl_required_api_version;
+}