feat: introduce --fail-on-ingest-warnings to detect, and treat as errors, warnings from ingest pipeline

[pkoutsovasilis]

This PR intoduces --fail-on-ingest-warnings that instructs elastic-package to check for warnings on the log of elasticsearch when running the pipeline test.

For example, clone integration repo at this SHA elastic/integrations@389e006

With the code of this PR running the pipeline test inside packages/cisco_asa results in the following:

$ elastic-package test pipeline --fail-on-ingest-warnings
Run pipeline tests for the package
2024/04/09 15:36:29  WARN ingest pipeline warning: character class has '-' without escape
2024/04/09 15:36:29  WARN ingest pipeline warning: nested repeat operator '+' and '?' was replaced with '*' in regular expression /(?:(?<CISCO_DOMAIN_USER:_temp_.cisco.source_username>((?:(LOCAL\\)?((?:\b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b))\\)?))?(?:[^,$)]+)?)\$?\)?(?:(, *((?:((?<NUMBER:_temp_.cisco.source_user_security_group_tag>(?:(?:(?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))))))):(?:((?<WORD:_temp_.cisco.source_user_security_group_tag_name>\b\w+\b)))|(?:((?<NUMBER:_temp_.cisco.source_user_security_group_tag>(?:(?:(?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+))))))))))|((?:((?<NUMBER:_temp_.cisco.source_user_security_group_tag>(?:(?:(?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))))))):(?:((?<WORD:_temp_.cisco.source_user_security_group_tag_name>\b\w+\b))))))|(?:(?<CISCO_DOMAIN_USER:_temp_.cisco.source_username>((?:(LOCAL\\)?((?:\b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b))\\)?))?(?:[^,$)]+)?)\$?\)?)/
2024/04/09 15:36:29  WARN ingest pipeline warning: nested repeat operator '+' and '?' was replaced with '*' in regular expression /(?:(?<CISCO_DOMAIN_USER:_temp_.cisco.destination_username>((?:(LOCAL\\)?((?:\b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b))\\)?))?(?:[^,$)]+)?)\$?\)?(?:(, *((?:((?<NUMBER:_temp_.cisco.destination_user_security_group_tag>(?:(?:(?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))))))):(?:((?<WORD:_temp_.cisco.destination_user_security_group_tag_name>\b\w+\b)))|(?:((?<NUMBER:_temp_.cisco.destination_user_security_group_tag>(?:(?:(?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+))))))))))|((?:((?<NUMBER:_temp_.cisco.destination_user_security_group_tag>(?:(?:(?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))))))):(?:((?<WORD:_temp_.cisco.destination_user_security_group_tag_name>\b\w+\b))))))|(?:(?<CISCO_DOMAIN_USER:_temp_.cisco.destination_username>((?:(LOCAL\\)?((?:\b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b))\\)?))?(?:[^,$)]+)?)\$?\)?)/
2024/04/09 15:36:29  WARN ingest pipeline warning: nested repeat operator '+' and '?' was replaced with '*' in regular expression /Teardown (?:.*?) (?<NOTSPACE:network.transport>\S+) translation from (?<NOTCOLON:_temp_.cisco.source_interface>[^:]*):(?<IPORHOST:source.address>(?:(?:(?:(?:((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?)|(?:(?<![0-9])(?:(?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5]))(?![0-9]))))|(?:\b(?:[0-9A-Za-z][0-9A-Za-z-_]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-_]{0,62}))*(\.?|\b))))/(?<NUMBER:source.port>(?:(?:(?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+))))))(\s*\((?<CISCO_USER:_temp_.cisco.source_username>(?:\*\*\*\*\*|(?:(?:LOCAL\\)?((?:\b(?:[0-9A-Za-z][0-9A-Za-z-_]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-_]{0,62}))*(\.?|\b))\\)?(?:(?:[a-zA-Z0-9._-]+))?\$?(?:@(?:\b(?:[0-9A-Za-z][0-9A-Za-z-_]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-_]{0,62}))*(\.?|\b)))?(?:, *(?:(?:(?:(?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))))))?(?:(?:(?:(?:(?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))))):(?:.*?))?)))\))? to (?<NOTCOLON:_temp_.cisco.destination_interface>[^:]*):(?<IP:destination.address>(?:(?:((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?)|(?:(?<![0-9])(?:(?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5]))(?![0-9]))))/(?<NUMBER:destination.port>(?:(?:(?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))))) duration (?<DURATION:_temp_.duration_hms>(?:(?:[+-]?(?:[0-9]+))):(?:(?:[0-5][0-9])):(?:(?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?)))/
--- Test results for package: cisco_asa - START ---
FAILURE DETAILS:
cisco_asa/log (service elasticsearch logs):



╭───────────┬─────────────┬───────────┬──────────────────────────────┬──────────────────────────────────────────────┬──────────────╮
│ PACKAGE   │ DATA STREAM │ TEST TYPE │ TEST NAME                    │ RESULT                                       │ TIME ELAPSED │
├───────────┼─────────────┼───────────┼──────────────────────────────┼──────────────────────────────────────────────┼──────────────┤
│ cisco_asa │ log         │ pipeline  │ test-additional-messages.log │ PASS                                         │  54.837916ms │
│ cisco_asa │ log         │ pipeline  │ test-anyconnect-messages.log │ PASS                                         │   8.908667ms │
│ cisco_asa │ log         │ pipeline  │ test-asa-fix.log             │ PASS                                         │   9.146291ms │
│ cisco_asa │ log         │ pipeline  │ test-asa-missing-groups.log  │ PASS                                         │   8.268542ms │
│ cisco_asa │ log         │ pipeline  │ test-asa.log                 │ PASS                                         │  112.75425ms │
│ cisco_asa │ log         │ pipeline  │ test-dap-records.log         │ PASS                                         │   4.776375ms │
│ cisco_asa │ log         │ pipeline  │ test-filtered.log            │ PASS                                         │   5.504208ms │
│ cisco_asa │ log         │ pipeline  │ test-hostnames.log           │ PASS                                         │   4.312959ms │
│ cisco_asa │ log         │ pipeline  │ test-non-canonical.log       │ PASS                                         │  11.926584ms │
│ cisco_asa │ log         │ pipeline  │ test-not-ip.log              │ PASS                                         │   6.044875ms │
│ cisco_asa │ log         │ pipeline  │ test-sample.log              │ PASS                                         │  47.160333ms │
│ cisco_asa │ log         │ pipeline  │ test-sgt-tag-name.log        │ PASS                                         │     24.326ms │
│ cisco_asa │ log         │ pipeline  │ test-sip.log                 │ PASS                                         │     5.7505ms │
│ cisco_asa │ log         │ pipeline  │ (service elasticsearch logs) │ FAIL: detected ingest pipeline warnings: 503 │ 1.764223625s │
╰───────────┴─────────────┴───────────┴──────────────────────────────┴──────────────────────────────────────────────┴──────────────╯
--- Test results for package: cisco_asa - END   ---

However, the code at main completes the test without any error. Having such a flag that we can opt in and check for warning is useful to prevent issues, such as this one in the future.

Closes #1754

[taylor-swanson]

It would be nice to surface these warnings all the time, not just when --fail-on-ingest-warnings is given. For new integrations, this would be great, but I could see this being "annoying" for existing ones and could cause lots of spam in CI logs.

The problem here is I didn't even know these logs existed in the first place. What's worse is they are discovered by end users and not devs. Even if we had a way to disable them for elastic-package (like other validations), the warnings will still surface for users.

[jsoriano]

It would be nice to surface these warnings all the time, not just when --fail-on-ingest-warnings is given.

+1, it'd be nice if we can just fail always on these errors, as they are actual issues. We introduced in the past some checks based on errors found in elastic-agent logs (#1256).

@pkoutsovasilis could you try to always enable this check and test with all integrations (adding a comment test integrations) here? If not many of them fail maybe we can go on and always enable this check.

[jsoriano]

test integrations

[elasticmachine]

Created or updated PR in integrations repository to test this version. Check elastic/integrations#9561

[pkoutsovasilis]

Overall integrations testing do not look that bad https://buildkite.com/elastic/integrations/builds/10324 although there is an integration that is not detected (cisco_ios). Just pushed a commit that hopefully address this

[pkoutsovasilis]

test integrations

[elasticmachine]

Created or updated PR in integrations repository to test this version. Check elastic/integrations#9561

[pkoutsovasilis]

test integrations

[elasticmachine]

Created or updated PR in integrations repository to test this version. Check elastic/integrations#9561

[pkoutsovasilis]

@jsoriano any idea why cisco_ftd in the integration test has a different log format compared to others like this

elasticsearch-1  | {"type": "server", "timestamp": "2024-04-10T12:54:26,235Z", "level": "WARN", "component": "o.e.i.c.GrokProcessor", "cluster.name": "elasticsearch", "node.name": "9e39afe7080b", "message": "character class has '-' without escape", "cluster.uuid": "WbdhEzxGRo6c9Td8pW646g", "node.id": "AATj6t7cTCqOrA8G2NT40Q"  }
elasticsearch-1  | {"type": "server", "timestamp": "2024-04-10T12:54:26,289Z", "level": "WARN", "component": "o.e.i.c.GrokProcessor", "cluster.name": "elasticsearch", "node.name": "9e39afe7080b", "message": "character class has '-' without escape", "cluster.uuid": "WbdhEzxGRo6c9Td8pW646g", "node.id": "AATj6t7cTCqOrA8G2NT40Q"  }
elasticsearch-1  | {"type": "server", "timestamp": "2024-04-10T12:54:26,292Z", "level": "WARN", "component": "o.e.i.c.GrokProcessor", "cluster.name": "elasticsearch", "node.name": "9e39afe7080b", "message": "character class has '-' without escape", "cluster.uuid": "WbdhEzxGRo6c9Td8pW646g", "node.id": "AATj6t7cTCqOrA8G2NT40Q"  }

do we need to support such a format in our log parsing? 🙂

[pkoutsovasilis]

test integrations

[pkoutsovasilis]

test integrations

[elasticmachine]

Created or updated PR in integrations repository to test this version. Check elastic/integrations#9561

[jsoriano]

any idea why cisco_ftd in the integration test has a different log format compared to others like this

Not sure if this is the reason, but cisco_ftd supports 7.16, so maybe something changed in 8.x regarding logs.

do we need to support such a format in our log parsing? 🙂

If the reason is the version of Elasticsearch, in principle we need to support since 7.14, the first version where Fleet was GA. So we need to support any log formats used between these versions. Or if possible, other approach could be to configure Elasticsearch to use the same logging format independently of the version.

[pkoutsovasilis]

test integrations

[elasticmachine]

Created or updated PR in integrations repository to test this version. Check elastic/integrations#9561

[pkoutsovasilis]

test integrations

[elasticmachine]

Created or updated PR in integrations repository to test this version. Check elastic/integrations#9561

[pkoutsovasilis]

@jsoriano all integrations having failures related to this feature (pipeline warnings) are fixed and I think this is now good to fly with this check always enabled. integrations run

[jsoriano]

/test

[jsoriano]

@pkoutsovasilis there are some tests consistently failing here, could you take a look to them?
It may be a matter of adding a new error to the file of expected errors for cisco_asa. It was expecting 3 errors but it looks like there are 4 now.

[jsoriano]

Here is the related part of the log about the failure: https://buildkite.com/elastic/elastic-package/builds/2822#018ee670-b8c2-4475-86a2-567e849247e6/126-547

[pkoutsovasilis]

Here is the related part of the log about the failure: https://buildkite.com/elastic/elastic-package/builds/2822#018ee670-b8c2-4475-86a2-567e849247e6/126-547

sure thing @jsoriano , thanks for catching that

[pkoutsovasilis]

🤦

regular expression has redundant nested repeat operator * /AAA user (?<DATA:_temp_.cisco.aaa_type>.*?) Successful((?:\s*))?: server =((?:\s*)+)?(?<IP:destination.address>(?:(?:((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)?)|(?:(?<![0-9])(?:(?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])[.](?:[0-1]?[0-9]{1,2}|2[0-4][0-9]|25[0-5]))(?![0-9])))) [:,] [Uu]ser = (?<CISCO_USER:source.user.name>((LOCAL\\)?((?:\b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b))\\)?(?:[a-zA-Z0-9._-]+)(@(?:\b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b)))?(, *(?:(?:(?:(?<![0-9.+-])(?>[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+)))))))?))/

[pkoutsovasilis]

/test

[jsoriano]

Thanks!

[elasticmachine]

💚 Build Succeeded

• Buildkite Build
• Commit: b3e1a72

History

• 💚 Build #2832 succeeded b3da911
• 💔 Build #2831 failed 0b51e62
• 💔 Build #2827 failed 95a7819
• 💔 Build #2825 failed 942060b
• 💔 Build #2822 failed b771542
• 💔 Build #2818 failed b771542