diff --git a/packages/ti_recordedfuture/_dev/build/docs/README.md b/packages/ti_recordedfuture/_dev/build/docs/README.md index ffb91cc66fd..8e0b04d3d80 100644 --- a/packages/ti_recordedfuture/_dev/build/docs/README.md +++ b/packages/ti_recordedfuture/_dev/build/docs/README.md @@ -10,6 +10,13 @@ from multiple entities, it's necessary to define one integration for each. Alternatively, it's also possible to use the integration to fetch custom Fusion files by supplying the URL to the CSV file as the _Custom_ _URL_ configuration option. +### Expiration of Indicators of Compromise (IOCs) +The ingested IOCs expire after certain duration. An [Elastic Transform](https://www.elastic.co/guide/en/elasticsearch/reference/current/transforms.html) is created to faciliate only active IOCs be available to the end users. This transform creates a destination index named `logs-ti_recordedfuture_latest.threat` which only contains active and unexpired IOCs. When setting up indicator match rules, use this latest destination index to avoid false positives from expired IOCs. Please read [ILM Policy](#ilm-policy) below which is added to avoid unbounded growth on source `.ds-logs-ti_recordedfuture.threat-*` indices. + +### ILM Policy +To facilitate IOC expiration, source datastream-backed indices `.ds-logs-ti_recordedfuture.threat-*` are allowed to contain duplicates from each polling interval. ILM policy is added to these source indices so it doesn't lead to unbounded growth. This means data in these source indices will be deleted after `5 days` from ingested date. + + **NOTE:** For large risklist downloads, adjust the timeout setting so that the Agent has enough time to download and process the risklist. {{event "threat"}} diff --git a/packages/ti_recordedfuture/changelog.yml b/packages/ti_recordedfuture/changelog.yml index cc0f696dd37..27724db8db5 100644 --- a/packages/ti_recordedfuture/changelog.yml +++ b/packages/ti_recordedfuture/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "1.9.0" + changes: + - description: Support for IoC Expiration + type: enhancement + link: https://github.com/elastic/integrations/issues/5459 - version: "1.8.0" changes: - description: Add a new flag to enable request tracing diff --git a/packages/ti_recordedfuture/data_stream/threat/_dev/test/pipeline/test-domain-default.log-expected.json b/packages/ti_recordedfuture/data_stream/threat/_dev/test/pipeline/test-domain-default.log-expected.json index 6082eb333d0..6b1b482330a 100644 --- a/packages/ti_recordedfuture/data_stream/threat/_dev/test/pipeline/test-domain-default.log-expected.json +++ b/packages/ti_recordedfuture/data_stream/threat/_dev/test/pipeline/test-domain-default.log-expected.json @@ -80,6 +80,7 @@ "Timestamp": "2021-12-29T07:12:02.455Z" } ], + "name": "xohrikvjhiu.eu", "risk_string": "5/45" }, "tags": [ @@ -193,6 +194,7 @@ "Timestamp": "2021-12-29T07:21:52.303Z" } ], + "name": "wgwuhauaqcrx.com", "risk_string": "6/45" }, "tags": [ @@ -305,6 +307,7 @@ "Timestamp": "2021-12-29T07:16:05.007Z" } ], + "name": "wbmpvebw.com", "risk_string": "6/45" }, "tags": [ @@ -405,6 +408,7 @@ "Timestamp": "2021-12-29T06:40:44.358Z" } ], + "name": "ckgryagcibbcf.com", "risk_string": "5/45" }, "tags": [ @@ -504,6 +508,7 @@ "Timestamp": "2021-12-29T06:46:28.155Z" } ], + "name": "jpuityvakjgg.com", "risk_string": "5/45" }, "tags": [ @@ -603,6 +608,7 @@ "Timestamp": "2021-12-29T06:40:30.778Z" } ], + "name": "jexgpprgph.com", "risk_string": "5/45" }, "tags": [ @@ -702,6 +708,7 @@ "Timestamp": "2021-12-29T06:34:06.062Z" } ], + "name": "cascotqhij.com", "risk_string": "5/45" }, "tags": [ @@ -801,6 +808,7 @@ "Timestamp": "2021-12-29T06:45:21.381Z" } ], + "name": "npcvnorvyhelagx.com", "risk_string": "5/45" }, "tags": [ @@ -900,6 +908,7 @@ "Timestamp": "2021-12-29T06:35:26.677Z" } ], + "name": "uxlyihgvfnqcrfcf.com", "risk_string": "5/45" }, "tags": [ diff --git a/packages/ti_recordedfuture/data_stream/threat/_dev/test/pipeline/test-hash-default.log-expected.json b/packages/ti_recordedfuture/data_stream/threat/_dev/test/pipeline/test-hash-default.log-expected.json index 885fa47d77e..c5cbef58a91 100644 --- a/packages/ti_recordedfuture/data_stream/threat/_dev/test/pipeline/test-hash-default.log-expected.json +++ b/packages/ti_recordedfuture/data_stream/threat/_dev/test/pipeline/test-hash-default.log-expected.json @@ -171,6 +171,7 @@ "Timestamp": "2020-07-11T09:55:23.000Z" } ], + "name": "38e992eb852ab0c4ac03955fb0dc9bb38e64010fdf9c05331d2b02b6e05689c2", "risk_string": "6/14" }, "tags": [ @@ -370,6 +371,7 @@ "Timestamp": "2021-03-08T00:00:00.000Z" } ], + "name": "c15abaf51e78ca56c0376522d699c978217bf041a3bd3c71d09193efa5717c71", "risk_string": "7/14" }, "tags": [ @@ -573,6 +575,7 @@ "Timestamp": "2021-12-18T00:20:04.000Z" } ], + "name": "b66db3a06c2955a9cb71a8718970c592", "risk_string": "5/14" }, "tags": [ @@ -951,6 +954,7 @@ "Timestamp": "2020-12-17T22:59:03.000Z" } ], + "name": "027cc450ef5f8c5f653329641ec1fed91f694e0d229928963b30f6b0d7d3a745", "risk_string": "8/14" }, "tags": [ @@ -1090,6 +1094,7 @@ "Timestamp": "2019-07-01T00:00:00.000Z" } ], + "name": "ad2ad0249fafe85877bc79a01e1afd1a44d983c064ad8cb5bc694d29d166217b", "risk_string": "5/14" }, "tags": [ @@ -1221,6 +1226,7 @@ "Timestamp": "2021-04-04T07:46:20.000Z" } ], + "name": "01ba1fb41632594997a41d0c3a911ae5b3034d566ebb991ef76ad76e6f9e283a", "risk_string": "5/14" }, "tags": [ @@ -1422,6 +1428,7 @@ "Timestamp": "2021-02-10T09:10:10.000Z" } ], + "name": "fecddb7f3fa478be4687ca542c0ecf232ec35a0c2418c8bfe4875686ec373c1e", "risk_string": "6/14" }, "tags": [ @@ -1546,6 +1553,7 @@ "Timestamp": "2020-10-13T10:46:31.000Z" } ], + "name": "a1d9cd6f189beff28a0a49b10f8fe4510128471f004b3e4283ddc7f78594906b", "risk_string": "3/14" }, "tags": [ @@ -1674,6 +1682,7 @@ "Timestamp": "2021-03-08T13:00:15.000Z" } ], + "name": "85aba198a0ba204e8549ea0c8980447249d30dece0d430e3f517315ad10f32ce", "risk_string": "5/14" }, "tags": [ diff --git a/packages/ti_recordedfuture/data_stream/threat/_dev/test/pipeline/test-httpjson.log-expected.json b/packages/ti_recordedfuture/data_stream/threat/_dev/test/pipeline/test-httpjson.log-expected.json index 8669c5c3ca8..11fdbebbd00 100644 --- a/packages/ti_recordedfuture/data_stream/threat/_dev/test/pipeline/test-httpjson.log-expected.json +++ b/packages/ti_recordedfuture/data_stream/threat/_dev/test/pipeline/test-httpjson.log-expected.json @@ -79,6 +79,7 @@ "Timestamp": "2021-12-29T07:12:02.455Z" } ], + "name": "xohrikvjhiu.eu", "risk_string": "5/45" }, "tags": [ @@ -192,6 +193,7 @@ "Timestamp": "2021-12-29T07:21:52.303Z" } ], + "name": "wgwuhauaqcrx.com", "risk_string": "6/45" }, "tags": [ @@ -304,6 +306,7 @@ "Timestamp": "2021-12-29T07:16:05.007Z" } ], + "name": "wbmpvebw.com", "risk_string": "6/45" }, "tags": [ @@ -404,6 +407,7 @@ "Timestamp": "2021-12-29T06:40:44.358Z" } ], + "name": "ckgryagcibbcf.com", "risk_string": "5/45" }, "tags": [ @@ -503,6 +507,7 @@ "Timestamp": "2021-12-29T06:46:28.155Z" } ], + "name": "jpuityvakjgg.com", "risk_string": "5/45" }, "tags": [ @@ -602,6 +607,7 @@ "Timestamp": "2021-12-29T06:40:30.778Z" } ], + "name": "jexgpprgph.com", "risk_string": "5/45" }, "tags": [ @@ -701,6 +707,7 @@ "Timestamp": "2021-12-29T06:34:06.062Z" } ], + "name": "cascotqhij.com", "risk_string": "5/45" }, "tags": [ @@ -800,6 +807,7 @@ "Timestamp": "2021-12-29T06:45:21.381Z" } ], + "name": "npcvnorvyhelagx.com", "risk_string": "5/45" }, "tags": [ @@ -899,6 +907,7 @@ "Timestamp": "2021-12-29T06:35:26.677Z" } ], + "name": "uxlyihgvfnqcrfcf.com", "risk_string": "5/45" }, "tags": [ @@ -998,6 +1007,7 @@ "Timestamp": "2021-12-29T06:48:58.905Z" } ], + "name": "bjfwfqviu.com", "risk_string": "5/45" }, "tags": [ @@ -1192,6 +1202,7 @@ "Timestamp": "2020-07-11T09:55:23.000Z" } ], + "name": "38e992eb852ab0c4ac03955fb0dc9bb38e64010fdf9c05331d2b02b6e05689c2", "risk_string": "6/14" }, "tags": [ @@ -1391,6 +1402,7 @@ "Timestamp": "2021-03-08T00:00:00.000Z" } ], + "name": "c15abaf51e78ca56c0376522d699c978217bf041a3bd3c71d09193efa5717c71", "risk_string": "7/14" }, "tags": [ @@ -1594,6 +1606,7 @@ "Timestamp": "2021-12-18T00:20:04.000Z" } ], + "name": "b66db3a06c2955a9cb71a8718970c592", "risk_string": "5/14" }, "tags": [ @@ -1972,6 +1985,7 @@ "Timestamp": "2020-12-17T22:59:03.000Z" } ], + "name": "027cc450ef5f8c5f653329641ec1fed91f694e0d229928963b30f6b0d7d3a745", "risk_string": "8/14" }, "tags": [ @@ -2111,6 +2125,7 @@ "Timestamp": "2019-07-01T00:00:00.000Z" } ], + "name": "ad2ad0249fafe85877bc79a01e1afd1a44d983c064ad8cb5bc694d29d166217b", "risk_string": "5/14" }, "tags": [ @@ -2242,6 +2257,7 @@ "Timestamp": "2021-04-04T07:46:20.000Z" } ], + "name": "01ba1fb41632594997a41d0c3a911ae5b3034d566ebb991ef76ad76e6f9e283a", "risk_string": "5/14" }, "tags": [ @@ -2443,6 +2459,7 @@ "Timestamp": "2021-02-10T09:10:10.000Z" } ], + "name": "fecddb7f3fa478be4687ca542c0ecf232ec35a0c2418c8bfe4875686ec373c1e", "risk_string": "6/14" }, "tags": [ @@ -2567,6 +2584,7 @@ "Timestamp": "2020-10-13T10:46:31.000Z" } ], + "name": "a1d9cd6f189beff28a0a49b10f8fe4510128471f004b3e4283ddc7f78594906b", "risk_string": "3/14" }, "tags": [ @@ -2695,6 +2713,7 @@ "Timestamp": "2021-03-08T13:00:15.000Z" } ], + "name": "85aba198a0ba204e8549ea0c8980447249d30dece0d430e3f517315ad10f32ce", "risk_string": "5/14" }, "tags": [ @@ -2831,6 +2850,7 @@ "Timestamp": "2017-08-13T00:33:27.000Z" } ], + "name": "7531fcea7002c8b52a8d023d0f3bb938efb2cbfec91d2433694930b426d84865", "risk_string": "5/14" }, "tags": [ @@ -2934,6 +2954,7 @@ "Timestamp": "2021-12-29T02:11:16.663Z" } ], + "name": "67.43.156.12", "risk_string": "4/64" }, "tags": [ @@ -3048,6 +3069,7 @@ "Timestamp": "2021-12-24T08:07:09.925Z" } ], + "name": "67.43.156.13", "risk_string": "6/64" }, "tags": [ @@ -3161,6 +3183,7 @@ "Timestamp": "2021-12-28T18:45:41.875Z" } ], + "name": "67.43.156.14", "risk_string": "6/64" }, "tags": [ @@ -3329,6 +3352,7 @@ "Timestamp": "2021-12-27T19:00:49.975Z" } ], + "name": "67.43.156.15", "risk_string": "10/64" }, "tags": [ @@ -3410,6 +3434,7 @@ "Timestamp": "2021-12-28T22:05:35.688Z" } ], + "name": "67.43.156.12", "risk_string": "3/64" }, "tags": [ @@ -3532,6 +3557,7 @@ "Timestamp": "2021-12-29T06:21:27.731Z" } ], + "name": "67.43.156.13", "risk_string": "7/64" }, "tags": [ @@ -3624,6 +3650,7 @@ "Timestamp": "2021-12-28T18:42:08.923Z" } ], + "name": "67.43.156.14", "risk_string": "4/64" }, "tags": [ @@ -3771,6 +3798,7 @@ "Timestamp": "2021-12-22T04:10:08.558Z" } ], + "name": "67.43.156.15", "risk_string": "8/64" }, "tags": [ @@ -3891,6 +3919,7 @@ "Timestamp": "2021-12-29T02:00:05.439Z" } ], + "name": "67.43.156.12", "risk_string": "6/64" }, "tags": [ @@ -4024,6 +4053,7 @@ "Timestamp": "2021-12-29T07:00:21.416Z" } ], + "name": "67.43.156.13", "risk_string": "7/64" }, "tags": [ @@ -4113,6 +4143,7 @@ "Timestamp": "2021-07-10T00:00:00.000Z" } ], + "name": "http://144.34.179.162/a", "risk_string": "2/24" }, "tags": [ @@ -4222,6 +4253,7 @@ "Timestamp": "2021-12-29T07:08:29.105Z" } ], + "name": "http://adminsys.serveftp.com/nensa/fabio/ex/478632215/zer7855/nuns566623", "risk_string": "4/24" }, "tags": [ @@ -4303,6 +4335,7 @@ "Timestamp": "2021-12-17T00:00:00.000Z" } ], + "name": "http://3.145.115.94/zambo/groenhuyzen.exe", "risk_string": "2/24" }, "tags": [ @@ -4384,6 +4417,7 @@ "Timestamp": "2021-12-28T00:00:00.000Z" } ], + "name": "http://gxbrowser.net", "risk_string": "2/24" }, "tags": [ @@ -4474,6 +4508,7 @@ "Timestamp": "2021-12-29T07:07:42.477Z" } ], + "name": "https://881.000webhostapp.com/1.txt", "risk_string": "3/24" }, "tags": [ @@ -4573,6 +4608,7 @@ "Timestamp": "2021-12-29T06:34:00.698Z" } ], + "name": "http://comunicador.duckdns.org/catalista/lixo/index.php", "risk_string": "4/24" }, "tags": [ @@ -4656,6 +4692,7 @@ "Timestamp": "2021-12-28T00:00:00.000Z" } ], + "name": "https://www.jeanninecatddns.chickenkiller.com/signin-authflow", "risk_string": "3/24" }, "tags": [ @@ -4740,6 +4777,7 @@ "Timestamp": "2021-08-12T00:00:00.000Z" } ], + "name": "http://coollab.jp/dir/root/p/09908.js", "risk_string": "3/24" }, "tags": [ @@ -4830,6 +4868,7 @@ "Timestamp": "2021-01-25T00:00:00.000Z" } ], + "name": "https://blog.br0vvnn.io", "risk_string": "3/24" }, "tags": [ @@ -4907,6 +4946,7 @@ "Timestamp": "2021-12-27T00:00:00.000Z" } ], + "name": "http://init.icloud-analysis.com", "risk_string": "2/24" }, "tags": [ diff --git a/packages/ti_recordedfuture/data_stream/threat/_dev/test/pipeline/test-ip-default.log-expected.json b/packages/ti_recordedfuture/data_stream/threat/_dev/test/pipeline/test-ip-default.log-expected.json index 994e90a94ac..36e05912c19 100644 --- a/packages/ti_recordedfuture/data_stream/threat/_dev/test/pipeline/test-ip-default.log-expected.json +++ b/packages/ti_recordedfuture/data_stream/threat/_dev/test/pipeline/test-ip-default.log-expected.json @@ -67,6 +67,7 @@ "Timestamp": "2021-12-29T02:11:16.663Z" } ], + "name": "1.128.3.4", "risk_string": "4/64" }, "tags": [ @@ -166,6 +167,7 @@ "Timestamp": "2021-12-23T10:18:13.994Z" } ], + "name": "2a02:cf40:add:4002:91f2:a9b2:e09a:6fc6", "risk_string": "5/64" }, "tags": [ @@ -277,6 +279,7 @@ "Timestamp": "2021-12-24T08:07:09.925Z" } ], + "name": "175.16.199.1", "risk_string": "6/64" }, "tags": [ @@ -390,6 +393,7 @@ "Timestamp": "2021-12-28T18:45:41.875Z" } ], + "name": "216.160.83.57", "risk_string": "6/64" }, "tags": [ @@ -558,6 +562,7 @@ "Timestamp": "2021-12-27T19:00:49.975Z" } ], + "name": "216.160.83.61", "risk_string": "10/64" }, "tags": [ @@ -639,6 +644,7 @@ "Timestamp": "2021-12-28T22:05:35.688Z" } ], + "name": "81.2.69.143", "risk_string": "3/64" }, "tags": [ @@ -761,6 +767,7 @@ "Timestamp": "2021-12-29T06:21:27.731Z" } ], + "name": "81.2.69.144", "risk_string": "7/64" }, "tags": [ @@ -853,6 +860,7 @@ "Timestamp": "2021-12-28T18:42:08.923Z" } ], + "name": "81.2.69.145", "risk_string": "4/64" }, "tags": [ @@ -1000,6 +1008,7 @@ "Timestamp": "2021-12-22T04:10:08.558Z" } ], + "name": "81.2.69.193", "risk_string": "8/64" }, "tags": [ diff --git a/packages/ti_recordedfuture/data_stream/threat/_dev/test/pipeline/test-url-default.log-expected.json b/packages/ti_recordedfuture/data_stream/threat/_dev/test/pipeline/test-url-default.log-expected.json index c64841f3148..61b44916fbc 100644 --- a/packages/ti_recordedfuture/data_stream/threat/_dev/test/pipeline/test-url-default.log-expected.json +++ b/packages/ti_recordedfuture/data_stream/threat/_dev/test/pipeline/test-url-default.log-expected.json @@ -61,6 +61,7 @@ "Timestamp": "2021-07-10T00:00:00.000Z" } ], + "name": "http://144.34.179.162/a", "risk_string": "2/24" }, "tags": [ @@ -170,6 +171,7 @@ "Timestamp": "2021-12-29T07:08:29.105Z" } ], + "name": "http://adminsys.serveftp.com/nensa/fabio/ex/478632215/zer7855/nuns566623", "risk_string": "4/24" }, "tags": [ @@ -251,6 +253,7 @@ "Timestamp": "2021-12-17T00:00:00.000Z" } ], + "name": "http://3.145.115.94/zambo/groenhuyzen.exe", "risk_string": "2/24" }, "tags": [ @@ -332,6 +335,7 @@ "Timestamp": "2021-12-28T00:00:00.000Z" } ], + "name": "http://gxbrowser.net", "risk_string": "2/24" }, "tags": [ @@ -422,6 +426,7 @@ "Timestamp": "2021-12-29T07:07:42.477Z" } ], + "name": "https://881.000webhostapp.com/1.txt", "risk_string": "3/24" }, "tags": [ @@ -521,6 +526,7 @@ "Timestamp": "2021-12-29T06:34:00.698Z" } ], + "name": "http://comunicador.duckdns.org/catalista/lixo/index.php", "risk_string": "4/24" }, "tags": [ @@ -604,6 +610,7 @@ "Timestamp": "2021-12-28T00:00:00.000Z" } ], + "name": "https://www.jeanninecatddns.chickenkiller.com/signin-authflow", "risk_string": "3/24" }, "tags": [ @@ -688,6 +695,7 @@ "Timestamp": "2021-08-12T00:00:00.000Z" } ], + "name": "http://coollab.jp/dir/root/p/09908.js", "risk_string": "3/24" }, "tags": [ @@ -778,6 +786,7 @@ "Timestamp": "2021-01-25T00:00:00.000Z" } ], + "name": "https://blog.br0vvnn.io", "risk_string": "3/24" }, "tags": [ @@ -866,6 +875,7 @@ "Timestamp": "2021-01-25T00:00:00.000Z" } ], + "name": "example.net/%e3%83%9d%e3%82%b1%e3%83%a2%e3%83%b3-%e3%82%bb%e3%83%b3%e3%82%bf%e3%83%bc-%e6%a8%aa%e6%b5%9c-%e7%a7%bb%e8%bb%a2-%e3%81%aa%e3%81%9c/%e3%81%b5%e3%82%8b%e3%81%95%e3%81%a8-%e7%b4%8d%e7%a8%8e-%e3%83%88%e3%82%a4%e3%83%ac%e3%83%83%e3%83%88-%e3%83%9a%e3%83%bc%e3%83%91%e3%83%bc-%e9%82%84%e5%85%83-%e7%8e%87/%e6%9c%9d%e6%97%a5-%e6%96%b0%e8%81%9e-be-%e3%83%91%e3%82%ba%e3%83%a", "risk_string": "3/24" }, "tags": [ diff --git a/packages/ti_recordedfuture/data_stream/threat/_dev/test/system/test-fusion-file-download-config.yml b/packages/ti_recordedfuture/data_stream/threat/_dev/test/system/test-fusion-file-download-config.yml index 145a4ae105a..f9475f2867c 100644 --- a/packages/ti_recordedfuture/data_stream/threat/_dev/test/system/test-fusion-file-download-config.yml +++ b/packages/ti_recordedfuture/data_stream/threat/_dev/test/system/test-fusion-file-download-config.yml @@ -1,5 +1,4 @@ service: recordedfuture-csv-download-http -service_notify_signal: SIGHUP input: httpjson wait_for_data_timeout: 1m data_stream: diff --git a/packages/ti_recordedfuture/data_stream/threat/_dev/test/system/test-risklist-api-config.yml b/packages/ti_recordedfuture/data_stream/threat/_dev/test/system/test-risklist-api-config.yml index c75710df10b..c7065122a5b 100644 --- a/packages/ti_recordedfuture/data_stream/threat/_dev/test/system/test-risklist-api-config.yml +++ b/packages/ti_recordedfuture/data_stream/threat/_dev/test/system/test-risklist-api-config.yml @@ -1,5 +1,4 @@ service: recordedfuture-api-http -service_notify_signal: SIGHUP input: httpjson data_stream: vars: diff --git a/packages/ti_recordedfuture/data_stream/threat/elasticsearch/ilm/default_policy.json b/packages/ti_recordedfuture/data_stream/threat/elasticsearch/ilm/default_policy.json new file mode 100644 index 00000000000..68d2c5e57a6 --- /dev/null +++ b/packages/ti_recordedfuture/data_stream/threat/elasticsearch/ilm/default_policy.json @@ -0,0 +1,23 @@ +{ + "policy": { + "phases": { + "hot": { + "actions": { + "rollover": { + "max_age": "2d", + "max_size": "50gb" + }, + "set_priority": { + "priority": 100 + } + } + }, + "delete": { + "min_age": "3d", + "actions": { + "delete": {} + } + } + } + } +} \ No newline at end of file diff --git a/packages/ti_recordedfuture/data_stream/threat/elasticsearch/ingest_pipeline/default.yml b/packages/ti_recordedfuture/data_stream/threat/elasticsearch/ingest_pipeline/default.yml index aad702b2d08..5a60b4d2ca6 100644 --- a/packages/ti_recordedfuture/data_stream/threat/elasticsearch/ingest_pipeline/default.yml +++ b/packages/ti_recordedfuture/data_stream/threat/elasticsearch/ingest_pipeline/default.yml @@ -209,16 +209,6 @@ processors: field: error.message value: "Risk score `{{{ json.Risk }}}` cannot be converted to float: {{{ _ingest.on_failure_message }}}" -# -# Fingerprint event: _id = hash(dataset + indicator type + indicator value) -# - - fingerprint: - fields: - - event.dataset - - threat.indicator.type - - json.Name - target_field: "_id" - # # Save fields without an ECS mapping under `recordedfuture`. # @@ -226,6 +216,9 @@ processors: field: json.RiskString target_field: json.risk_string ignore_missing: true + - rename: + field: json.Name + target_field: json.name - rename: field: json target_field: recordedfuture diff --git a/packages/ti_recordedfuture/data_stream/threat/manifest.yml b/packages/ti_recordedfuture/data_stream/threat/manifest.yml index c2a4ead3ce8..a468152fa56 100644 --- a/packages/ti_recordedfuture/data_stream/threat/manifest.yml +++ b/packages/ti_recordedfuture/data_stream/threat/manifest.yml @@ -1,5 +1,6 @@ type: logs title: Recorded Future +ilm_policy: logs-ti_recordedfuture.threat-default_policy streams: - input: logfile enabled: false diff --git a/packages/ti_recordedfuture/data_stream/threat/sample_event.json b/packages/ti_recordedfuture/data_stream/threat/sample_event.json index 7580117e585..3a0790e9293 100644 --- a/packages/ti_recordedfuture/data_stream/threat/sample_event.json +++ b/packages/ti_recordedfuture/data_stream/threat/sample_event.json @@ -1,11 +1,11 @@ { - "@timestamp": "2023-04-24T01:13:37.004Z", + "@timestamp": "2023-05-03T17:51:08.713Z", "agent": { - "ephemeral_id": "32aafc9a-68e1-47fe-bef5-36d8890a898f", - "id": "c5e28d1b-c611-499c-9a49-08340eeaa6a0", + "ephemeral_id": "0bec0fc8-832a-4ac3-8e25-2805084a462d", + "id": "286609b3-211c-48aa-ae77-2d6182f4a2cd", "name": "docker-fleet-agent", "type": "filebeat", - "version": "8.6.2" + "version": "8.8.0" }, "data_stream": { "dataset": "ti_recordedfuture.threat", @@ -16,15 +16,15 @@ "version": "8.7.0" }, "elastic_agent": { - "id": "c5e28d1b-c611-499c-9a49-08340eeaa6a0", - "snapshot": false, - "version": "8.6.2" + "id": "286609b3-211c-48aa-ae77-2d6182f4a2cd", + "snapshot": true, + "version": "8.8.0" }, "event": { "agent_id_status": "verified", "category": "threat", "dataset": "ti_recordedfuture.threat", - "ingested": "2023-04-24T01:13:38Z", + "ingested": "2023-05-03T17:51:09Z", "kind": "enrichment", "risk_score": 87, "timezone": "+00:00", @@ -87,6 +87,7 @@ "Timestamp": "2021-07-10T00:00:00.000Z" } ], + "name": "http://144.34.179.162/a", "risk_string": "2/24" }, "tags": [ diff --git a/packages/ti_recordedfuture/docs/README.md b/packages/ti_recordedfuture/docs/README.md index 8023f10f638..919c50ef49c 100644 --- a/packages/ti_recordedfuture/docs/README.md +++ b/packages/ti_recordedfuture/docs/README.md @@ -10,19 +10,26 @@ from multiple entities, it's necessary to define one integration for each. Alternatively, it's also possible to use the integration to fetch custom Fusion files by supplying the URL to the CSV file as the _Custom_ _URL_ configuration option. +### Expiration of Indicators of Compromise (IOCs) +The ingested IOCs expire after certain duration. An [Elastic Transform](https://www.elastic.co/guide/en/elasticsearch/reference/current/transforms.html) is created to faciliate only active IOCs be available to the end users. This transform creates a destination index named `logs-ti_recordedfuture_latest.threat` which only contains active and unexpired IOCs. When setting up indicator match rules, use this latest destination index to avoid false positives from expired IOCs. Please read [ILM Policy](#ilm-policy) below which is added to avoid unbounded growth on source `.ds-logs-ti_recordedfuture.threat-*` indices. + +### ILM Policy +To facilitate IOC expiration, source datastream-backed indices `.ds-logs-ti_recordedfuture.threat-*` are allowed to contain duplicates from each polling interval. ILM policy is added to these source indices so it doesn't lead to unbounded growth. This means data in these source indices will be deleted after `5 days` from ingested date. + + **NOTE:** For large risklist downloads, adjust the timeout setting so that the Agent has enough time to download and process the risklist. An example event for `threat` looks as following: ```json { - "@timestamp": "2023-04-24T01:13:37.004Z", + "@timestamp": "2023-05-03T17:51:08.713Z", "agent": { - "ephemeral_id": "32aafc9a-68e1-47fe-bef5-36d8890a898f", - "id": "c5e28d1b-c611-499c-9a49-08340eeaa6a0", + "ephemeral_id": "0bec0fc8-832a-4ac3-8e25-2805084a462d", + "id": "286609b3-211c-48aa-ae77-2d6182f4a2cd", "name": "docker-fleet-agent", "type": "filebeat", - "version": "8.6.2" + "version": "8.8.0" }, "data_stream": { "dataset": "ti_recordedfuture.threat", @@ -33,15 +40,15 @@ An example event for `threat` looks as following: "version": "8.7.0" }, "elastic_agent": { - "id": "c5e28d1b-c611-499c-9a49-08340eeaa6a0", - "snapshot": false, - "version": "8.6.2" + "id": "286609b3-211c-48aa-ae77-2d6182f4a2cd", + "snapshot": true, + "version": "8.8.0" }, "event": { "agent_id_status": "verified", "category": "threat", "dataset": "ti_recordedfuture.threat", - "ingested": "2023-04-24T01:13:38Z", + "ingested": "2023-05-03T17:51:09Z", "kind": "enrichment", "risk_score": 87, "timezone": "+00:00", @@ -104,6 +111,7 @@ An example event for `threat` looks as following: "Timestamp": "2021-07-10T00:00:00.000Z" } ], + "name": "http://144.34.179.162/a", "risk_string": "2/24" }, "tags": [ diff --git a/packages/ti_recordedfuture/elasticsearch/transform/latest_ioc/fields/agent.yml b/packages/ti_recordedfuture/elasticsearch/transform/latest_ioc/fields/agent.yml new file mode 100644 index 00000000000..da4e652c53b --- /dev/null +++ b/packages/ti_recordedfuture/elasticsearch/transform/latest_ioc/fields/agent.yml @@ -0,0 +1,198 @@ +- name: cloud + title: Cloud + group: 2 + description: Fields related to the cloud or infrastructure the events are coming from. + footnote: 'Examples: If Metricbeat is running on an EC2 host and fetches data from its host, the cloud info contains the data about this machine. If Metricbeat runs on a remote machine outside the cloud and fetches data from a service running in the cloud, the field contains cloud data from the machine the service is running on.' + type: group + fields: + - name: account.id + level: extended + type: keyword + ignore_above: 1024 + description: 'The cloud account or organization id used to identify different entities in a multi-tenant environment. + + Examples: AWS account id, Google Cloud ORG Id, or other unique identifier.' + example: 666777888999 + - name: availability_zone + level: extended + type: keyword + ignore_above: 1024 + description: Availability zone in which this host is running. + example: us-east-1c + - name: instance.id + level: extended + type: keyword + ignore_above: 1024 + description: Instance ID of the host machine. + example: i-1234567890abcdef0 + - name: instance.name + level: extended + type: keyword + ignore_above: 1024 + description: Instance name of the host machine. + - name: machine.type + level: extended + type: keyword + ignore_above: 1024 + description: Machine type of the host machine. + example: t2.medium + - name: provider + level: extended + type: keyword + ignore_above: 1024 + description: Name of the cloud provider. Example values are aws, azure, gcp, or digitalocean. + example: aws + - name: region + level: extended + type: keyword + ignore_above: 1024 + description: Region in which this host is running. + example: us-east-1 + - name: project.id + type: keyword + description: Name of the project in Google Cloud. + - name: image.id + type: keyword + description: Image ID for the cloud instance. +- name: container + title: Container + group: 2 + description: 'Container fields are used for meta information about the specific container that is the source of information. + + These fields help correlate data based containers from any runtime.' + type: group + fields: + - name: id + level: core + type: keyword + ignore_above: 1024 + description: Unique container id. + - name: image.name + level: extended + type: keyword + ignore_above: 1024 + description: Name of the image the container was built on. + - name: labels + level: extended + type: object + object_type: keyword + description: Image labels. + - name: name + level: extended + type: keyword + ignore_above: 1024 + description: Container name. +- name: host + title: Host + group: 2 + description: 'A host is defined as a general computing instance. + + ECS host.* fields should be populated with details about the host on which the event happened, or from which the measurement was taken. Host types include hardware, virtual machines, Docker containers, and Kubernetes nodes.' + type: group + fields: + - name: architecture + level: core + type: keyword + ignore_above: 1024 + description: Operating system architecture. + example: x86_64 + - name: domain + level: extended + type: keyword + ignore_above: 1024 + description: 'Name of the domain of which the host is a member. + + For example, on Windows this could be the host''s Active Directory domain or NetBIOS domain name. For Linux this could be the domain of the host''s LDAP provider.' + example: CONTOSO + default_field: false + - name: hostname + level: core + type: keyword + ignore_above: 1024 + description: 'Hostname of the host. + + It normally contains what the `hostname` command returns on the host machine.' + - name: id + level: core + type: keyword + ignore_above: 1024 + description: 'Unique host id. + + As hostname is not always unique, use values that are meaningful in your environment. + + Example: The current usage of `beat.name`.' + - name: ip + level: core + type: ip + description: Host ip addresses. + - name: mac + level: core + type: keyword + ignore_above: 1024 + description: Host mac addresses. + - name: name + level: core + type: keyword + ignore_above: 1024 + description: 'Name of the host. + + It can contain what `hostname` returns on Unix systems, the fully qualified domain name, or a name specified by the user. The sender decides which value to use.' + - name: os.family + level: extended + type: keyword + ignore_above: 1024 + description: OS family (such as redhat, debian, freebsd, windows). + example: debian + - name: os.kernel + level: extended + type: keyword + ignore_above: 1024 + description: Operating system kernel version as a raw string. + example: 4.4.0-112-generic + - name: os.name + level: extended + type: keyword + ignore_above: 1024 + multi_fields: + - name: text + type: text + norms: false + default_field: false + description: Operating system name, without the version. + example: Mac OS X + - name: os.platform + level: extended + type: keyword + ignore_above: 1024 + description: Operating system platform (such centos, ubuntu, windows). + example: darwin + - name: os.version + level: extended + type: keyword + ignore_above: 1024 + description: Operating system version as a raw string. + example: 10.14.1 + - name: type + level: core + type: keyword + ignore_above: 1024 + description: 'Type of host. + + For Cloud providers this can be the machine type like `t2.medium`. If vm, this could be the container, for example, or other information meaningful in your environment.' + - name: containerized + type: boolean + description: > + If the host is a container. + + - name: os.build + type: keyword + example: "18D109" + description: > + OS build information. + + - name: os.codename + type: keyword + example: "stretch" + description: > + OS codename, if any. + diff --git a/packages/ti_recordedfuture/elasticsearch/transform/latest_ioc/fields/base-fields.yml b/packages/ti_recordedfuture/elasticsearch/transform/latest_ioc/fields/base-fields.yml new file mode 100644 index 00000000000..1fbc652b8a2 --- /dev/null +++ b/packages/ti_recordedfuture/elasticsearch/transform/latest_ioc/fields/base-fields.yml @@ -0,0 +1,31 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset name. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: event.module + type: constant_keyword + description: Event module + value: ti_recordedfuture +- name: event.dataset + type: constant_keyword + description: Event dataset + value: ti_recordedfuture.threat +- name: threat.feed.name + type: constant_keyword + description: Display friendly feed name + value: Recorded Future +# +# TODO: Add dashboard +# +#- name: threat.feed.dashboard_id +# type: constant_keyword +# description: Dashboard ID used for Kibana CTI UI +# value: recordedfuture-96fe1e60-4261-11ec-b7be-d3026acdf1cf +- name: "@timestamp" + type: date + description: Event timestamp. diff --git a/packages/ti_recordedfuture/elasticsearch/transform/latest_ioc/fields/beats.yml b/packages/ti_recordedfuture/elasticsearch/transform/latest_ioc/fields/beats.yml new file mode 100644 index 00000000000..cb44bb29442 --- /dev/null +++ b/packages/ti_recordedfuture/elasticsearch/transform/latest_ioc/fields/beats.yml @@ -0,0 +1,12 @@ +- name: input.type + type: keyword + description: Type of Filebeat input. +- name: log.flags + type: keyword + description: Flags for the log file. +- name: log.offset + type: long + description: Offset of the entry in the log file. +- name: log.file.path + type: keyword + description: Path to the log file. diff --git a/packages/ti_recordedfuture/elasticsearch/transform/latest_ioc/fields/ecs.yml b/packages/ti_recordedfuture/elasticsearch/transform/latest_ioc/fields/ecs.yml new file mode 100644 index 00000000000..380ba3cda52 --- /dev/null +++ b/packages/ti_recordedfuture/elasticsearch/transform/latest_ioc/fields/ecs.yml @@ -0,0 +1,70 @@ +- external: ecs + name: ecs.version +- external: ecs + name: message +- external: ecs + name: tags +- external: ecs + name: error.message +- external: ecs + name: event.severity +- external: ecs + name: event.category +- external: ecs + name: event.ingested +- external: ecs + name: event.created +- external: ecs + name: event.kind +- external: ecs + name: event.type +- external: ecs + name: event.original +- external: ecs + name: threat.indicator.first_seen +- external: ecs + name: threat.indicator.last_seen +- external: ecs + name: threat.indicator.type +- external: ecs + name: threat.indicator.ip +- external: ecs + name: threat.indicator.url.domain +- external: ecs + name: threat.indicator.url.full +- external: ecs + name: threat.indicator.url.extension +- external: ecs + name: threat.indicator.url.original +- external: ecs + name: threat.indicator.url.path +- external: ecs + name: threat.indicator.url.port +- external: ecs + name: threat.indicator.url.scheme +- external: ecs + name: threat.indicator.url.query +- external: ecs + name: threat.indicator.file.hash.md5 +- external: ecs + name: threat.indicator.file.hash.sha1 +- external: ecs + name: threat.indicator.file.hash.sha256 +- external: ecs + name: threat.indicator.file.hash.sha512 +- external: ecs + name: threat.indicator.email.address +- external: ecs + name: threat.indicator.provider +- external: ecs + name: threat.indicator.marking.tlp +- external: ecs + name: threat.indicator.confidence +- external: ecs + name: threat.indicator.as.number +- external: ecs + name: threat.indicator.as.organization.name +- external: ecs + name: threat.indicator.geo.location +- external: ecs + name: threat.indicator.geo.country_iso_code diff --git a/packages/ti_recordedfuture/elasticsearch/transform/latest_ioc/fields/fields.yml b/packages/ti_recordedfuture/elasticsearch/transform/latest_ioc/fields/fields.yml new file mode 100644 index 00000000000..732eee6f63f --- /dev/null +++ b/packages/ti_recordedfuture/elasticsearch/transform/latest_ioc/fields/fields.yml @@ -0,0 +1,30 @@ +- name: "event.created" + type: date + description: Time when event got created. +- name: "event.ingested" + type: date + description: Time when event gets ingested into Elasticsearch. +- name: "@timestamp" + type: date + description: Event timestamp. +- name: recordedfuture + type: group + description: > + Fields for Recorded Future Threat Intel + + fields: + - name: evidence_details + type: flattened + description: > + List of sightings used as evidence for this indicator. + + - name: name + type: keyword + description: > + Indicator value. + + - name: risk_string + type: keyword + description: > + Details of risk rules observed. + diff --git a/packages/ti_recordedfuture/elasticsearch/transform/latest_ioc/manifest.yml b/packages/ti_recordedfuture/elasticsearch/transform/latest_ioc/manifest.yml new file mode 100644 index 00000000000..79f3df8107b --- /dev/null +++ b/packages/ti_recordedfuture/elasticsearch/transform/latest_ioc/manifest.yml @@ -0,0 +1,18 @@ +start: true +destination_index_template: + settings: + index: + sort.field: + - "@timestamp" + sort.order: + - desc + mappings: + dynamic: true + _meta: {} + dynamic_templates: + - strings_as_keyword: + match_mapping_type: string + mapping: + ignore_above: 1024 + type: keyword + date_detection: false diff --git a/packages/ti_recordedfuture/elasticsearch/transform/latest_ioc/transform.yml b/packages/ti_recordedfuture/elasticsearch/transform/latest_ioc/transform.yml new file mode 100644 index 00000000000..9b3e96bf3c4 --- /dev/null +++ b/packages/ti_recordedfuture/elasticsearch/transform/latest_ioc/transform.yml @@ -0,0 +1,33 @@ +# Use of "*" to use all namespaces defined. +source: + index: + - "logs-ti_recordedfuture.threat-*" +# The version suffix on the dest.index should be incremented if a breaking change +# is made to the index mapping. You must also bump the fleet_transform_version +# for any change to this transform configuration to take effect. The old destination +# index is not automatically removed. We are dependent on https://github.com/elastic/package-spec/issues/523 to give +# us that ability in order to prevent having duplicate IoC data and prevent query +# time field type conflicts. +dest: + index: "logs-ti_recordedfuture_latest.threat-1" +latest: + unique_key: + - event.dataset + - threat.indicator.type + - recordedfuture.name + sort: "@timestamp" +description: Latest Recorded Future IoC data +frequency: 30s +sync: + time: + field: event.ingested + delay: 60s +retention_policy: + time: + field: event.ingested + # After internal discussion, 24 hours is a valid default retention period for Recorded Future. + max_age: 24h +_meta: + managed: true + # Bump this version to delete, reinstall, and restart the transform during package. + fleet_transform_version: 0.1.0 diff --git a/packages/ti_recordedfuture/manifest.yml b/packages/ti_recordedfuture/manifest.yml index 488231f19f3..74769136328 100644 --- a/packages/ti_recordedfuture/manifest.yml +++ b/packages/ti_recordedfuture/manifest.yml @@ -1,6 +1,6 @@ name: ti_recordedfuture title: Recorded Future -version: "1.8.0" +version: "1.9.0" release: ga description: Ingest threat intelligence indicators from Recorded Future risk lists with Elastic Agent. type: integration @@ -8,7 +8,7 @@ format_version: 1.0.0 license: basic categories: ["security", "threat_intel"] conditions: - kibana.version: ^8.7.1 + kibana.version: ^8.8.0 icons: - src: /img/logo.svg title: Recorded Future