TimescaleDB output

README.md

@@ -40,6 +40,7 @@ It works as a single endpoint for as many as you want `Falco` instances :
 - [**Prometheus**](https://prometheus.io/) (for both events and monitoring of `falcosidekick`)
 - [**Wavefront**](https://www.wavefront.com)
 - [**Spyderbat**](https://www.spyderbat.com)
+- [**TimescaleDB**](https://www.timescale.com/)
 
 ### Alerting
 
@@ -535,6 +536,15 @@ spyderbat:
   # source: "falcosidekick" # Spyderbat source ID, max 32 characters (default: "falcosidekick")
   # sourcedescription: "" # Spyderbat source description and display name if not empty, max 256 characters
   # minimumpriority: "debug" # minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default)
+
+timescaledb:
+  # host: "" # TimescaleDB host, if not empty, TImescaleDB output is enabled
+  # port: "5432" # TimescaleDB port (default: 5432)
+  # user: "postgres" # Username to authenticate with TimescaleDB (default: postgres)
+  # password: "postgres" # Password to authenticate with TimescaleDB (default: postgres)
+  # database: "" # TimescaleDB database used
+  # hypertablename: "falco_events" # Hypertable to store data events (default: falco_events) See TimescaleDB setup for more info
+  # minimumpriority: "" # minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default)
 ```
 
 Usage :
@@ -989,6 +999,13 @@ order is
 - **SPYDERBAT_SOURCE**: Spyderbat source ID, max 32 characters (default: "falcosidekick")
 - **SPYDERBAT_SOURCEDESCRIPTION**: Spyderbat source description and display name if not empty, max 256 characters
 - **SPYDERBAT_MINIMUMPRIORITY**: minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default)
+- **TIMESCALEDB_HOST**: TimescaleDB host, if not empty, TImescaleDB output is enabled
+- **TIMESCALEDB_PORT**: TimescaleDB port (default: 5432)
+- **TIMESCALEDB_USER**: Username to authenticate with TimescaleDB (default: postgres)
+- **TIMESCALEDB_PASSWORD**: Password to authenticate with TimescaleDB (default: postgres)
+- **TIMESCALEDB_DATABASE**: TimescaleDB database used
+- **TIMESCALEDB_HYPERTABLENAME**: Hypertable to store data events (default: falco_events) See TimescaleDB setup for more info
+- **TIMESCALEDB_MINIMUMPRIORITY**: minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default)
 
 #### Slack/Rocketchat/Mattermost/Googlechat Message Formatting
 
@@ -1144,6 +1161,23 @@ permissions to access the resources you selected to use, like `SQS`, `Lambda`,
 }
 ```
 
+### TimescaleDB setup
+
+To use TimescaleDB you should create the Hypertable first, following this example
+
+```sql
+CREATE TABLE falco_events (
+	time TIMESTAMPTZ NOT NULL,
+	rule TEXT,
+	priority VARCHAR(20),
+	source VARCHAR(20),
+	output TEXT
+);
+SELECT create_hypertable('falco_events', 'time');
+```
+
+The name from the table should match with the `hypertable` output configuration.
+
 ## Examples
 
 Run you daemon and try (from Falco's documentation) :

config.go

@@ -374,6 +374,14 @@ func getConfig() *types.Configuration {
 	v.SetDefault("Spyderbat.SourceDescription", "")
 	v.SetDefault("Spyderbat.MinimumPriority", "")
 
+	v.SetDefault("TimescaleDB.Host", "")
+	v.SetDefault("TimescaleDB.Port", "5432")
+	v.SetDefault("TimescaleDB.User", "postgres")
+	v.SetDefault("TimescaleDB.Password", "postgres")
+	v.SetDefault("TimescaleDB.Database", "falcosidekick")
+	v.SetDefault("TimescaleDB.Hypertable", "falcosidekick_events")
+	v.SetDefault("TimescaleDB.MinimumPriority", "")
+
 	v.SetEnvKeyReplacer(strings.NewReplacer(".", "_"))
 	v.AutomaticEnv()
 	if *configFile != "" {
@@ -549,6 +557,7 @@ func getConfig() *types.Configuration {
 	c.Zincsearch.MinimumPriority = checkPriority(c.Zincsearch.MinimumPriority)
 	c.NodeRed.MinimumPriority = checkPriority(c.NodeRed.MinimumPriority)
 	c.Gotify.MinimumPriority = checkPriority(c.Gotify.MinimumPriority)
+	c.TimescaleDB.MinimumPriority = checkPriority(c.TimescaleDB.MinimumPriority)
 
 	c.Slack.MessageFormatTemplate = getMessageFormatTemplate("Slack", c.Slack.MessageFormat)
 	c.Rocketchat.MessageFormatTemplate = getMessageFormatTemplate("Rocketchat", c.Rocketchat.MessageFormat)

config_example.yaml

@@ -363,4 +363,13 @@ spyderbat:
   # apiurl: "https://api.spyderbat.com" # Spyderbat API url (default: "https://api.spyderbat.com")
   # source: "falcosidekick" # Spyderbat source ID, max 32 characters (default: "falcosidekick")
   # sourcedescription: "" # Spyderbat source description and display name if not empty, max 256 characters
-  # minimumpriority: "debug" # minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default)
+  # minimumpriority: "debug" # minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default)
+
+timescaledb:
+  # host: "" # TimescaleDB host, if not empty, TImescaleDB output is enabled
+  # port: "5432" # TimescaleDB port (default: 5432)
+  # user: "postgres" # Username to authenticate with TimescaleDB (default: postgres)
+  # password: "postgres" # Password to authenticate with TimescaleDB (default: postgres)
+  # database: "" # TimescaleDB database used
+  # hypertablename: "falco_events" # Hypertable to store data events (default: falco_events) See TimescaleDB setup for more info
+  # minimumpriority: "" # minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default)

go.mod

@@ -77,6 +77,10 @@ require (
 	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/imdario/mergo v0.3.13 // indirect
+	github.com/jackc/pgpassfile v1.0.0 // indirect
+	github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
+	github.com/jackc/pgx/v5 v5.0.4 // indirect
+	github.com/jackc/puddle/v2 v2.0.0 // indirect
 	github.com/jmespath/go-jmespath v0.4.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/jpillora/backoff v1.0.0 // indirect
@@ -119,7 +123,7 @@ require (
 	golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde // indirect
 	golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 // indirect
 	golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 // indirect
-	golang.org/x/text v0.3.7 // indirect
+	golang.org/x/text v0.3.8 // indirect
 	golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect
 	golang.org/x/tools v0.1.12 // indirect
 	golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect

go.sum

@@ -508,6 +508,16 @@ github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH
 github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk=
 github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
+github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
+github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg=
+github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E=
+github.com/jackc/pgx/v5 v5.0.3 h1:4flM5ecR/555F0EcnjdaZa6MhBU+nr0QbZIo5vaKjuM=
+github.com/jackc/pgx/v5 v5.0.3/go.mod h1:JBbvW3Hdw77jKl9uJrEDATUZIFM2VFPzRq4RWIhkF4o=
+github.com/jackc/pgx/v5 v5.0.4 h1:r5O6y84qHX/z/HZV40JBdx2obsHz7/uRj5b+CcYEdeY=
+github.com/jackc/pgx/v5 v5.0.4/go.mod h1:U0ynklHtgg43fue9Ly30w3OCSTDPlXjig9ghrNGaguQ=
+github.com/jackc/puddle/v2 v2.0.0 h1:Kwk/AlLigcnZsDssc3Zun1dk1tAtQNPaBBxBHWn0Mjc=
+github.com/jackc/puddle/v2 v2.0.0/go.mod h1:itE7ZJY8xnoo0JqJEpSMprN0f+NQkMCuEV/N9j8h0oc=
 github.com/jinzhu/gorm v0.0.0-20160404144928-5174cc5c242a/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo=
 github.com/jinzhu/inflection v0.0.0-20170102125226-1c35d901db3d/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
 github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8=
@@ -1127,6 +1137,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

handlers.go

@@ -352,4 +352,8 @@ func forwardEvent(falcopayload types.FalcoPayload) {
 	if config.Spyderbat.OrgUID != "" && (falcopayload.Priority >= types.Priority(config.Spyderbat.MinimumPriority) || falcopayload.Rule == testRule) {
 		go spyderbatClient.SpyderbatPost(falcopayload)
 	}
+
+	if config.TimescaleDB.Host != "" && (falcopayload.Priority >= types.Priority(config.TimescaleDB.MinimumPriority) || falcopayload.Rule == testRule) {
+		go timescaleDBClient.TimescaleDBPost(falcopayload)
+	}
 }

main.go

@@ -60,6 +60,7 @@ var (
 	zincsearchClient    *outputs.Client
 	gotifyClient        *outputs.Client
 	spyderbatClient     *outputs.Client
+	timescaleDBClient   *outputs.Client
 
 	statsdClient, dogstatsdClient *statsd.Client
 	config                        *types.Configuration
@@ -617,6 +618,17 @@ func init() {
 		}
 	}
 
+	if config.TimescaleDB.Host != "" {
+		var err error
+		timescaleDBClient, err = outputs.NewTimescaleDBClient(config, stats, promStats, statsdClient, dogstatsdClient)
+		if err != nil {
+			config.TimescaleDB.Host = ""
+			log.Printf("[ERROR] : TimescaleDB - %v\n", err)
+		} else {
+			outputs.EnabledOutputs = append(outputs.EnabledOutputs, "TimescaleDB")
+		}
+	}
+
 	log.Printf("[INFO]  : Falco Sidekick version: %s\n", GetVersionInfo().GitVersion)
 	log.Printf("[INFO]  : Enabled Outputs : %s\n", outputs.EnabledOutputs)
 

outputs/client.go

@@ -31,6 +31,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 
 	mqtt "github.com/eclipse/paho.mqtt.golang"
+	timescaledb "github.com/jackc/pgx/v5/pgxpool"
 
 	"github.com/falcosecurity/falcosidekick/types"
 )
@@ -112,6 +113,7 @@ type Client struct {
 	WavefrontSender   *wavefront.Sender
 	Crdclient         *crdClient.Clientset
 	MQTTClient        mqtt.Client
+	TimescaleDBClient *timescaledb.Pool
 }
 
 // NewClient returns a new output.Client for accessing the different API.

outputs/timescaledb.go

@@ -0,0 +1,61 @@
+package outputs
+
+import (
+	"context"
+	"fmt"
+	"log"
+
+	"github.com/DataDog/datadog-go/statsd"
+	"github.com/falcosecurity/falcosidekick/types"
+	"github.com/jackc/pgx/v5/pgxpool"
+)
+
+func NewTimescaleDBClient(config *types.Configuration, stats *types.Statistics, promStats *types.PromStatistics,
+	statsdClient, dogstatsdClient *statsd.Client) (*Client, error) {
+
+	ctx := context.Background()
+	connStr := fmt.Sprintf(
+		"postgres://%s:%s@%s:%s/%s",
+		config.TimescaleDB.User,
+		config.TimescaleDB.Password,
+		config.TimescaleDB.Host,
+		config.TimescaleDB.Port,
+		config.TimescaleDB.Database,
+	)
+	connPool, err := pgxpool.New(ctx, connStr)
+	if err != nil {
+		log.Printf("[ERROR] : TimescaleDB - %v\n", err)
+		return nil, ErrClientCreation
+	}
+
+	return &Client{
+		OutputType:        "TimescaleDB",
+		Config:            config,
+		TimescaleDBClient: connPool,
+		Stats:             stats,
+		PromStats:         promStats,
+		StatsdClient:      statsdClient,
+		DogstatsdClient:   dogstatsdClient,
+	}, nil
+}
+
+func (c *Client) TimescaleDBPost(falcopayload types.FalcoPayload) {
+	c.Stats.TimescaleDB.Add(Total, 1)
+
+	hypertable := c.Config.TimescaleDB.HypertableName
+	queryInsertData := fmt.Sprintf("INSERT INTO %s (time, rule, priority, source, output) VALUES ($1, $2, $3, $4, $5)", hypertable)
+
+	var ctx = context.Background()
+	_, err := c.TimescaleDBClient.Exec(ctx, queryInsertData, falcopayload.Time, falcopayload.Rule, falcopayload.Priority.String(), falcopayload.Source, falcopayload.Output)
+	if err != nil {
+		go c.CountMetric(Outputs, 1, []string{"output:timescaledb", "status:error"})
+		c.Stats.TimescaleDB.Add(Error, 1)
+		c.PromStats.Outputs.With(map[string]string{"destination": "timescaledb", "status": Error}).Inc()
+		log.Printf("[ERROR] : TimescaleDB - %v\n", err)
+		return
+	}
+
+	go c.CountMetric(Outputs, 1, []string{"output:timescaledb", "status:ok"})
+	c.Stats.TimescaleDB.Add(OK, 1)
+	c.PromStats.Outputs.With(map[string]string{"destination": "timescaledb", "status": OK}).Inc()
+}

stats.go

@@ -73,6 +73,7 @@ func getInitStats() *types.Statistics {
 		NodeRed:           getOutputNewMap("nodered"),
 		Zincsearch:        getOutputNewMap("zincsearch"),
 		Gotify:            getOutputNewMap("gotify"),
+		TimescaleDB:       getOutputNewMap("timescaledb"),
 	}
 	stats.Falco.Add(outputs.Emergency, 0)
 	stats.Falco.Add(outputs.Alert, 0)

types/types.go

@@ -93,6 +93,7 @@ type Configuration struct {
 	Zincsearch         zincsearchOutputConfig
 	Gotify             gotifyOutputConfig
 	Spyderbat          SpyderbatConfig
+	TimescaleDB        TimescaleDBConfig
 }
 
 // SlackOutputConfig represents parameters for Slack
@@ -580,6 +581,16 @@ type SpyderbatConfig struct {
 	MinimumPriority   string
 }
 
+type TimescaleDBConfig struct {
+	Host            string
+	Port            string
+	User            string
+	Password        string
+	Database        string
+	HypertableName  string
+	MinimumPriority string
+}
+
 // Statistics is a struct to store stastics
 type Statistics struct {
 	Requests          *expvar.Map
@@ -637,6 +648,7 @@ type Statistics struct {
 	Zincsearch        *expvar.Map
 	Gotify            *expvar.Map
 	Spyderbat         *expvar.Map
+	TimescaleDB       *expvar.Map
 }
 
 // PromStatistics is a struct to store prometheus metrics