WIP: Elasticsearch Matcher #28
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,137 @@ | ||
| package elasticsearch | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "strings" | ||
| log "github.com/sirupsen/logrus" | ||
| "encoding/json" | ||
| "errors" | ||
| ) | ||
|
|
||
| func FilterByUuidPresence(alerts []ElasticsearchQueryHit, uuid string) []ElasticsearchQueryHit { | ||
| var filteredAlerts []ElasticsearchQueryHit | ||
| var containsUuid bool | ||
| for _,alert := range alerts { | ||
| containsUuid = false | ||
| for _,v := range alert.Source { | ||
| if strings.Contains(v.(string), uuid) { | ||
| containsUuid = true | ||
| break | ||
| } | ||
| } | ||
| if containsUuid { | ||
| filteredAlerts = append(filteredAlerts, alert) | ||
| } | ||
| } | ||
| return filteredAlerts | ||
| } | ||
|
|
||
| func StripHTTPStatusCode(response string) (string, error) { | ||
| index := strings.Index(response, "{") | ||
| if index != -1 { | ||
| return response[index:], nil | ||
| } else { | ||
| return "", errors.New("No '{' found in Elasticsearch query response") | ||
| } | ||
| } | ||
|
|
||
| func RetrieveAlerts(m *ElasticsearchAlertGeneratedAssertion, uuidField, ruleName string) ([]ElasticsearchQueryHit, error) { | ||
| // The alias for the Elasticsearch index where alerts are stored | ||
| const ALERT_INDEX string = ".siem-signals-default" | ||
| // Construct the query necessary to find the alert | ||
| query := ` | ||
| { | ||
| "_source": [ "%s" ], | ||
| "query": { | ||
| "bool": { | ||
| "filter": [ | ||
| { "range": { "@timestamp": { "gte": "now-3d" }}}, | ||
| { "term": { "kibana.alert.rule.name": "%s" }}, | ||
| { "term": { "kibana.alert.workflow_status": "open" }} | ||
| ] | ||
| } | ||
| } | ||
| }` | ||
| // Template in the field we expect to find the UUID in, and the rule we hope was triggered. | ||
| query = fmt.Sprintf(query, uuidField, ruleName) | ||
| // Query the Elasticsearch API | ||
| res, err := m.AlertAPI.Search( | ||
| m.AlertAPI.Search.WithIndex(ALERT_INDEX), | ||
| m.AlertAPI.Search.WithBody(strings.NewReader(query)), | ||
| ) | ||
| if err != nil { | ||
| log.Fatal("Error while running Elasticsearch query") | ||
| return nil, err | ||
| } | ||
| // Parse the response | ||
| strippedResponse, err := StripHTTPStatusCode(res.String()) | ||
| if err != nil { | ||
| log.Fatal("Error while stripping prepended HTTP status code") | ||
|
There was a problem hiding this comment. Do you want to include the error here? |
||
| return nil, err | ||
| } | ||
| var data ElasticsearchQueryResponse | ||
| if err := json.Unmarshal([]byte(strippedResponse), &data); err != nil { | ||
| log.Fatal("Error unmarshalling JSON string into ElasticsearchQueryResponse struct") | ||
|
There was a problem hiding this comment. You may want to log the error here so it is easier to debug |
||
| return nil, err | ||
| } | ||
|
|
||
| return data.Hits.Hits, nil | ||
| } | ||
|
|
||
| func (m *ElasticsearchAlertGeneratedAssertion) HasExpectedAlert(detonationUuid string) (bool, error) { | ||
| log.Infof("Searching for open alerts for rule: %s with UUID: %s in field: %s", m.AlertFilter.RuleName, detonationUuid, m.AlertFilter.UuidField) | ||
| alerts, err := RetrieveAlerts(m, m.AlertFilter.UuidField, m.AlertFilter.RuleName) | ||
| if err != nil { | ||
| log.Fatal("Failed to retrieve alerts") | ||
| return false, err | ||
| } | ||
| // Filter the alerts, is the one we're looking for here? | ||
| alerts = FilterByUuidPresence(alerts, detonationUuid) | ||
| if len(alerts) == 1 { | ||
| log.Info("One open alert found") | ||
| m.AlertId = alerts[0].ID | ||
| m.Index = alerts[0].Index | ||
| return true, nil | ||
| } | ||
| if len(alerts) > 1 { | ||
| // TODO: It may well be desirable for a suspicious event to trigger multiple alerts | ||
| // In future ElasticsearchAlertGeneratedAssertion.AlertFilter should be a list, capable | ||
| // of matching and closing multiple alerts associated with a single event. | ||
| log.Errorf("More than one alert found") | ||
| return false, nil | ||
| } | ||
| log.Warnf("No alerts found") | ||
| return false, nil | ||
| } | ||
|
|
||
| func (m *ElasticsearchAlertGeneratedAssertion) String() string { | ||
| return fmt.Sprintf("Elasticsearch alert '%s'", m.AlertFilter.RuleName) | ||
| } | ||
|
|
||
| func (m *ElasticsearchAlertGeneratedAssertion) Cleanup(detonationUuid string) error { | ||
| log.Infof("Closing alert for detonation: %s, for rule: %s with AlertId: %s in Index: %s", detonationUuid, m.AlertFilter.RuleName, m.AlertId, m.Index) | ||
| // If HasExpectedAlert() executed properly then m.AlertId ought to be set with the ID we need | ||
| if m.AlertId == "" { | ||
| return errors.New("AlertId not set, cannot close alert") | ||
| } | ||
| // We can query via the .siem-signals-default alias, however this isn't the actual index the document is in. | ||
| // To write to the index we need the actual index ID. Fortunately that data is in the document and we should | ||
| // have written that also when we ran HasExpectedAlert(). | ||
| if m.Index == "" { | ||
| return errors.New("Index not set, cannot close alert") | ||
| } | ||
| update_request_body := ` | ||
| { | ||
| "doc": { | ||
| "kibana.alert.workflow_status": "closed" | ||
| } | ||
| }` | ||
| resp, err := m.AlertAPI.Update(m.Index, m.AlertId, strings.NewReader(update_request_body)) | ||
| log.Info("Logging the update API response:\n",resp, "\n") | ||
| if err != nil { | ||
| log.Errorf("Error while trying to update document: %s", m.AlertId) | ||
| return err | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,114 @@ | ||
| package elasticsearch | ||
|
|
||
| import ( | ||
| "os" | ||
| "time" | ||
| "github.com/cenkalti/backoff/v4" | ||
| "strings" | ||
| "fmt" | ||
|
|
||
| log "github.com/sirupsen/logrus" | ||
| es "github.com/elastic/go-elasticsearch/v8" | ||
| ) | ||
|
|
||
| type ElasticsearchQueryResponse struct { | ||
| Hits struct { | ||
| Total struct { | ||
| Value int `json:"value"` | ||
| } `json:"total"` | ||
| Hits []ElasticsearchQueryHit `json:"hits"` | ||
| } `json:"hits"` | ||
| } | ||
|
|
||
| type ElasticsearchQueryHit struct { | ||
| Index string `json:"_index"` | ||
| ID string `json:"_id"` | ||
| Source map[string]interface{} `json:"_source"` | ||
| } | ||
|
|
||
| type ElasticsearchAlertFilter struct { | ||
| RuleName string `yaml:"rule-name"` | ||
| UuidField string `yaml:"uuid-field"` | ||
| } | ||
|
|
||
| type ElasticsearchAlertGeneratedAssertion struct { | ||
| AlertAPI es.Client | ||
| AlertFilter *ElasticsearchAlertFilter | ||
| AlertId string | ||
| Index string | ||
| } | ||
|
|
||
| func ElasticsearchAlert(ruleName, uuidField string) *ElasticsearchAlertGeneratedAssertion { | ||
| retryBackoff := backoff.NewExponentialBackOff() | ||
| // New Elasticsearch client | ||
| esClient, err := es.NewClient(es.Config{ | ||
| Addresses: []string{os.Getenv("ELASTICSEARCH_URL")}, | ||
| Username: os.Getenv("ELASTICSEARCH_USERNAME"), | ||
| Password: os.Getenv("ELASTICSEARCH_PASSWORD"), | ||
| // Retry on 429 TooManyRequests statuses | ||
| RetryOnStatus: []int{502, 503, 504, 429}, | ||
| // Configure the backoff function | ||
| RetryBackoff: func(i int) time.Duration { | ||
| if i == 1 { | ||
| retryBackoff.Reset() | ||
| } | ||
| return retryBackoff.NextBackOff() | ||
| }, | ||
| // Retry up to 5 attempts | ||
| MaxRetries: 5, | ||
| }) | ||
| if err != nil { | ||
| log.Fatalf("failed to create Elasticsearch client: %w", err) | ||
| } | ||
| info, err := esClient.Info() | ||
| if err != nil { | ||
| log.Fatalf("failed to get Elasticsearch cluster info: %w", err) | ||
| } | ||
| log.Info("Elasticsearch cluster info:\n", info.String()) | ||
| return &ElasticsearchAlertGeneratedAssertion{ | ||
| AlertAPI: *esClient, | ||
| AlertFilter: &ElasticsearchAlertFilter{RuleName: ruleName, UuidField: uuidField}, | ||
| } | ||
| } | ||
|
|
||
|
|
||
| // Dumping Ground | ||
| func CreateIndex(m *ElasticsearchAlertGeneratedAssertion, index string) { | ||
| mapping := ` | ||
| { | ||
| "settings": { | ||
| "number_of_shards": 1 | ||
| }, | ||
| "mappings": { | ||
| "properties": { | ||
| "field1": { | ||
| "type": "text" | ||
| }, | ||
| "date": { | ||
| "type": "date" | ||
| } | ||
| } | ||
| } | ||
| }` | ||
| res, err := m.AlertAPI.Indices.Create( | ||
| index, | ||
| m.AlertAPI.Indices.Create.WithBody(strings.NewReader(mapping)), | ||
| ) | ||
| if err != nil { | ||
| log.Fatal(err) | ||
| } | ||
| log.Println(res) | ||
| } | ||
|
|
||
| func WriteToIndex(m *ElasticsearchAlertGeneratedAssertion, index string) { | ||
| entry := ` | ||
| { | ||
| "field1": "helloo there abc1234", | ||
| "@timestamp": "%s" | ||
| }` | ||
| res, err := m.AlertAPI.Index(index, strings.NewReader(fmt.Sprintf(entry, time.Now().Format("2006/01/02 15:04:05")))) | ||
| if err != nil { | ||
| log.Fatal(err) | ||
| } | ||
| log.Println(res) | ||
| } | ||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Spacing in the for loops and moving the variable inside the loop