diff --git a/README.md b/README.md index a5c80245a..9ed30490e 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,7 @@ It works as a single endpoint for as many as you want `Falco` instances : - [**Prometheus**](https://prometheus.io/) (for both events and monitoring of `falcosidekick`) - [**Wavefront**](https://www.wavefront.com) - [**Spyderbat**](https://www.spyderbat.com) +- [**TimescaleDB**](https://www.timescale.com/) ### Alerting @@ -535,6 +536,15 @@ spyderbat: # source: "falcosidekick" # Spyderbat source ID, max 32 characters (default: "falcosidekick") # sourcedescription: "" # Spyderbat source description and display name if not empty, max 256 characters # minimumpriority: "debug" # minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default) + +timescaledb: + # host: "" # TimescaleDB host, if not empty, TImescaleDB output is enabled + # port: "5432" # TimescaleDB port (default: 5432) + # user: "postgres" # Username to authenticate with TimescaleDB (default: postgres) + # password: "postgres" # Password to authenticate with TimescaleDB (default: postgres) + # database: "" # TimescaleDB database used + # hypertablename: "falco_events" # Hypertable to store data events (default: falco_events) See TimescaleDB setup for more info + # minimumpriority: "" # minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default) ``` Usage : @@ -989,6 +999,13 @@ order is - **SPYDERBAT_SOURCE**: Spyderbat source ID, max 32 characters (default: "falcosidekick") - **SPYDERBAT_SOURCEDESCRIPTION**: Spyderbat source description and display name if not empty, max 256 characters - **SPYDERBAT_MINIMUMPRIORITY**: minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default) +- **TIMESCALEDB_HOST**: TimescaleDB host, if not empty, TImescaleDB output is enabled +- **TIMESCALEDB_PORT**: TimescaleDB port (default: 5432) +- **TIMESCALEDB_USER**: Username to authenticate with TimescaleDB (default: postgres) +- **TIMESCALEDB_PASSWORD**: Password to authenticate with TimescaleDB (default: postgres) +- **TIMESCALEDB_DATABASE**: TimescaleDB database used +- **TIMESCALEDB_HYPERTABLENAME**: Hypertable to store data events (default: falco_events) See TimescaleDB setup for more info +- **TIMESCALEDB_MINIMUMPRIORITY**: minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default) #### Slack/Rocketchat/Mattermost/Googlechat Message Formatting @@ -1144,6 +1161,23 @@ permissions to access the resources you selected to use, like `SQS`, `Lambda`, } ``` +### TimescaleDB setup + +To use TimescaleDB you should create the Hypertable first, following this example + +```sql +CREATE TABLE falco_events ( + time TIMESTAMPTZ NOT NULL, + rule TEXT, + priority VARCHAR(20), + source VARCHAR(20), + output TEXT +); +SELECT create_hypertable('falco_events', 'time'); +``` + +The name from the table should match with the `hypertable` output configuration. + ## Examples Run you daemon and try (from Falco's documentation) : diff --git a/config.go b/config.go index fcc6e39ae..4f373f2ca 100644 --- a/config.go +++ b/config.go @@ -374,6 +374,14 @@ func getConfig() *types.Configuration { v.SetDefault("Spyderbat.SourceDescription", "") v.SetDefault("Spyderbat.MinimumPriority", "") + v.SetDefault("TimescaleDB.Host", "") + v.SetDefault("TimescaleDB.Port", "5432") + v.SetDefault("TimescaleDB.User", "postgres") + v.SetDefault("TimescaleDB.Password", "postgres") + v.SetDefault("TimescaleDB.Database", "falcosidekick") + v.SetDefault("TimescaleDB.Hypertable", "falcosidekick_events") + v.SetDefault("TimescaleDB.MinimumPriority", "") + v.SetEnvKeyReplacer(strings.NewReplacer(".", "_")) v.AutomaticEnv() if *configFile != "" { @@ -549,6 +557,7 @@ func getConfig() *types.Configuration { c.Zincsearch.MinimumPriority = checkPriority(c.Zincsearch.MinimumPriority) c.NodeRed.MinimumPriority = checkPriority(c.NodeRed.MinimumPriority) c.Gotify.MinimumPriority = checkPriority(c.Gotify.MinimumPriority) + c.TimescaleDB.MinimumPriority = checkPriority(c.TimescaleDB.MinimumPriority) c.Slack.MessageFormatTemplate = getMessageFormatTemplate("Slack", c.Slack.MessageFormat) c.Rocketchat.MessageFormatTemplate = getMessageFormatTemplate("Rocketchat", c.Rocketchat.MessageFormat) diff --git a/config_example.yaml b/config_example.yaml index ac9a34c17..2f9daeb9b 100644 --- a/config_example.yaml +++ b/config_example.yaml @@ -363,4 +363,13 @@ spyderbat: # apiurl: "https://api.spyderbat.com" # Spyderbat API url (default: "https://api.spyderbat.com") # source: "falcosidekick" # Spyderbat source ID, max 32 characters (default: "falcosidekick") # sourcedescription: "" # Spyderbat source description and display name if not empty, max 256 characters - # minimumpriority: "debug" # minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default) \ No newline at end of file + # minimumpriority: "debug" # minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default) + +timescaledb: + # host: "" # TimescaleDB host, if not empty, TImescaleDB output is enabled + # port: "5432" # TimescaleDB port (default: 5432) + # user: "postgres" # Username to authenticate with TimescaleDB (default: postgres) + # password: "postgres" # Password to authenticate with TimescaleDB (default: postgres) + # database: "" # TimescaleDB database used + # hypertablename: "falco_events" # Hypertable to store data events (default: falco_events) See TimescaleDB setup for more info + # minimumpriority: "" # minimum priority of event for using this output, order is emergency|alert|critical|error|warning|notice|informational|debug or "" (default) diff --git a/go.mod b/go.mod index 62f0409ed..bbb22a58d 100644 --- a/go.mod +++ b/go.mod @@ -77,6 +77,10 @@ require ( github.com/gorilla/websocket v1.5.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/imdario/mergo v0.3.13 // indirect + github.com/jackc/pgpassfile v1.0.0 // indirect + github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect + github.com/jackc/pgx/v5 v5.0.4 // indirect + github.com/jackc/puddle/v2 v2.0.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/jpillora/backoff v1.0.0 // indirect @@ -119,7 +123,7 @@ require ( golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde // indirect golang.org/x/sys v0.0.0-20220829200755-d48e67d00261 // indirect golang.org/x/term v0.0.0-20220722155259-a9ba230a4035 // indirect - golang.org/x/text v0.3.7 // indirect + golang.org/x/text v0.3.8 // indirect golang.org/x/time v0.0.0-20220722155302-e5dcc9cfc0b9 // indirect golang.org/x/tools v0.1.12 // indirect golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect diff --git a/go.sum b/go.sum index 7564b60a6..10db5a119 100644 --- a/go.sum +++ b/go.sum @@ -508,6 +508,16 @@ github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgx/v5 v5.0.3 h1:4flM5ecR/555F0EcnjdaZa6MhBU+nr0QbZIo5vaKjuM= +github.com/jackc/pgx/v5 v5.0.3/go.mod h1:JBbvW3Hdw77jKl9uJrEDATUZIFM2VFPzRq4RWIhkF4o= +github.com/jackc/pgx/v5 v5.0.4 h1:r5O6y84qHX/z/HZV40JBdx2obsHz7/uRj5b+CcYEdeY= +github.com/jackc/pgx/v5 v5.0.4/go.mod h1:U0ynklHtgg43fue9Ly30w3OCSTDPlXjig9ghrNGaguQ= +github.com/jackc/puddle/v2 v2.0.0 h1:Kwk/AlLigcnZsDssc3Zun1dk1tAtQNPaBBxBHWn0Mjc= +github.com/jackc/puddle/v2 v2.0.0/go.mod h1:itE7ZJY8xnoo0JqJEpSMprN0f+NQkMCuEV/N9j8h0oc= github.com/jinzhu/gorm v0.0.0-20160404144928-5174cc5c242a/go.mod h1:Vla75njaFJ8clLU1W44h34PjIkijhjHIYnZxMqCdxqo= github.com/jinzhu/inflection v0.0.0-20170102125226-1c35d901db3d/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/now v1.0.1/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= @@ -1127,6 +1137,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/handlers.go b/handlers.go index 3f79949db..6e710ad39 100644 --- a/handlers.go +++ b/handlers.go @@ -352,4 +352,8 @@ func forwardEvent(falcopayload types.FalcoPayload) { if config.Spyderbat.OrgUID != "" && (falcopayload.Priority >= types.Priority(config.Spyderbat.MinimumPriority) || falcopayload.Rule == testRule) { go spyderbatClient.SpyderbatPost(falcopayload) } + + if config.TimescaleDB.Host != "" && (falcopayload.Priority >= types.Priority(config.TimescaleDB.MinimumPriority) || falcopayload.Rule == testRule) { + go timescaleDBClient.TimescaleDBPost(falcopayload) + } } diff --git a/main.go b/main.go index 250514e66..1237d0cf1 100644 --- a/main.go +++ b/main.go @@ -60,6 +60,7 @@ var ( zincsearchClient *outputs.Client gotifyClient *outputs.Client spyderbatClient *outputs.Client + timescaleDBClient *outputs.Client statsdClient, dogstatsdClient *statsd.Client config *types.Configuration @@ -617,6 +618,17 @@ func init() { } } + if config.TimescaleDB.Host != "" { + var err error + timescaleDBClient, err = outputs.NewTimescaleDBClient(config, stats, promStats, statsdClient, dogstatsdClient) + if err != nil { + config.TimescaleDB.Host = "" + log.Printf("[ERROR] : TimescaleDB - %v\n", err) + } else { + outputs.EnabledOutputs = append(outputs.EnabledOutputs, "TimescaleDB") + } + } + log.Printf("[INFO] : Falco Sidekick version: %s\n", GetVersionInfo().GitVersion) log.Printf("[INFO] : Enabled Outputs : %s\n", outputs.EnabledOutputs) diff --git a/outputs/client.go b/outputs/client.go index 245a207a7..0e20f21ad 100644 --- a/outputs/client.go +++ b/outputs/client.go @@ -31,6 +31,7 @@ import ( "k8s.io/client-go/kubernetes" mqtt "github.com/eclipse/paho.mqtt.golang" + timescaledb "github.com/jackc/pgx/v5/pgxpool" "github.com/falcosecurity/falcosidekick/types" ) @@ -112,6 +113,7 @@ type Client struct { WavefrontSender *wavefront.Sender Crdclient *crdClient.Clientset MQTTClient mqtt.Client + TimescaleDBClient *timescaledb.Pool } // NewClient returns a new output.Client for accessing the different API. diff --git a/outputs/timescaledb.go b/outputs/timescaledb.go new file mode 100644 index 000000000..75b6801df --- /dev/null +++ b/outputs/timescaledb.go @@ -0,0 +1,61 @@ +package outputs + +import ( + "context" + "fmt" + "log" + + "github.com/DataDog/datadog-go/statsd" + "github.com/falcosecurity/falcosidekick/types" + "github.com/jackc/pgx/v5/pgxpool" +) + +func NewTimescaleDBClient(config *types.Configuration, stats *types.Statistics, promStats *types.PromStatistics, + statsdClient, dogstatsdClient *statsd.Client) (*Client, error) { + + ctx := context.Background() + connStr := fmt.Sprintf( + "postgres://%s:%s@%s:%s/%s", + config.TimescaleDB.User, + config.TimescaleDB.Password, + config.TimescaleDB.Host, + config.TimescaleDB.Port, + config.TimescaleDB.Database, + ) + connPool, err := pgxpool.New(ctx, connStr) + if err != nil { + log.Printf("[ERROR] : TimescaleDB - %v\n", err) + return nil, ErrClientCreation + } + + return &Client{ + OutputType: "TimescaleDB", + Config: config, + TimescaleDBClient: connPool, + Stats: stats, + PromStats: promStats, + StatsdClient: statsdClient, + DogstatsdClient: dogstatsdClient, + }, nil +} + +func (c *Client) TimescaleDBPost(falcopayload types.FalcoPayload) { + c.Stats.TimescaleDB.Add(Total, 1) + + hypertable := c.Config.TimescaleDB.HypertableName + queryInsertData := fmt.Sprintf("INSERT INTO %s (time, rule, priority, source, output) VALUES ($1, $2, $3, $4, $5)", hypertable) + + var ctx = context.Background() + _, err := c.TimescaleDBClient.Exec(ctx, queryInsertData, falcopayload.Time, falcopayload.Rule, falcopayload.Priority.String(), falcopayload.Source, falcopayload.Output) + if err != nil { + go c.CountMetric(Outputs, 1, []string{"output:timescaledb", "status:error"}) + c.Stats.TimescaleDB.Add(Error, 1) + c.PromStats.Outputs.With(map[string]string{"destination": "timescaledb", "status": Error}).Inc() + log.Printf("[ERROR] : TimescaleDB - %v\n", err) + return + } + + go c.CountMetric(Outputs, 1, []string{"output:timescaledb", "status:ok"}) + c.Stats.TimescaleDB.Add(OK, 1) + c.PromStats.Outputs.With(map[string]string{"destination": "timescaledb", "status": OK}).Inc() +} diff --git a/stats.go b/stats.go index 497fe962d..d69c14209 100644 --- a/stats.go +++ b/stats.go @@ -73,6 +73,7 @@ func getInitStats() *types.Statistics { NodeRed: getOutputNewMap("nodered"), Zincsearch: getOutputNewMap("zincsearch"), Gotify: getOutputNewMap("gotify"), + TimescaleDB: getOutputNewMap("timescaledb"), } stats.Falco.Add(outputs.Emergency, 0) stats.Falco.Add(outputs.Alert, 0) diff --git a/types/types.go b/types/types.go index df8bfdacf..682dc41a4 100644 --- a/types/types.go +++ b/types/types.go @@ -93,6 +93,7 @@ type Configuration struct { Zincsearch zincsearchOutputConfig Gotify gotifyOutputConfig Spyderbat SpyderbatConfig + TimescaleDB TimescaleDBConfig } // SlackOutputConfig represents parameters for Slack @@ -580,6 +581,16 @@ type SpyderbatConfig struct { MinimumPriority string } +type TimescaleDBConfig struct { + Host string + Port string + User string + Password string + Database string + HypertableName string + MinimumPriority string +} + // Statistics is a struct to store stastics type Statistics struct { Requests *expvar.Map @@ -637,6 +648,7 @@ type Statistics struct { Zincsearch *expvar.Map Gotify *expvar.Map Spyderbat *expvar.Map + TimescaleDB *expvar.Map } // PromStatistics is a struct to store prometheus metrics