Skip to content

Commit

Permalink
Fix pr comments (#11)
Browse files Browse the repository at this point in the history 
* Fix PR comments

* Update with PR review comments

* Fix LGTM warnings of publish tool

* Fix bug in unit test

* Change to ip to match yang model

* Add changes per peer review
  • Loading branch information
@zbud-msft
zbud-msft authored Aug 25, 2022
1 parent b6cf34b commit 2d09ad4
Show file tree
Hide file tree
Showing 6 changed files with 139 additions and 88 deletions.
2 changes: 1 addition & 1 deletion dockers/docker-fpm-frr/bgp_regex.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
{
"tag": "bgp-state",
"regex": "Peer .default\\|([0-9a-f:.]*[0-9a-f]*). admin state is set to .(up|down).",
"params": [ "peer_ip", "status" ]
"params": [ "ip", "status" ]
}
]

50 changes: 40 additions & 10 deletions src/sonic-eventd/rsyslog_plugin/rsyslog_plugin.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,50 +25,80 @@ bool RsyslogPlugin::onMessage(string msg, lua_State* luaState) {
}
}

void parseParams(vector<string> params, vector<EventParam>& eventParams) {
for(long unsigned int i = 0; i < params.size(); i++) {
if(params[i].empty()) {
SWSS_LOG_ERROR("Empty param provided in regex file\n");
continue;
}
EventParam ep = EventParam();
auto delimPos = params[i].find(':');
if(delimPos == string::npos) { // no lua code
ep.paramName = params[i];
} else {
ep.paramName = params[i].substr(0, delimPos);
ep.luaCode = params[i].substr(delimPos + 1);
if(ep.luaCode.empty()) {
SWSS_LOG_ERROR("Lua code missing after :\n");
}
}
eventParams.push_back(ep);
}
}

bool RsyslogPlugin::createRegexList() {
fstream regexFile;
json jsonList = json::array();
regexFile.open(m_regexPath, ios::in);
if (!regexFile) {
SWSS_LOG_ERROR("No such path exists: %s for source %s\n", m_regexPath.c_str(), m_moduleName.c_str());
return false;
}
try {
regexFile >> m_parser->m_regexList;
regexFile >> jsonList;
} catch (invalid_argument& iaException) {
SWSS_LOG_ERROR("Invalid JSON file: %s, throws exception: %s\n", m_regexPath.c_str(), iaException.what());
return false;
}

string regexString;
string timestampRegex = "^([a-zA-Z]{3})?\\s*([0-9]{1,2})?\\s*([0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{0,6})?\\s*";
regex expression;
vector<RegexStruct> regexList;

for(long unsigned int i = 0; i < m_parser->m_regexList.size(); i++) {
for(long unsigned int i = 0; i < jsonList.size(); i++) {
RegexStruct rs = RegexStruct();
vector<EventParam> eventParams;
try {
string timestampRegex = "^([a-zA-Z]{3})?\\s*([0-9]{1,2})?\\s*([0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{0,6})?\\s*";
string eventRegex = m_parser->m_regexList[i]["regex"];
string eventRegex = jsonList[i]["regex"];
regexString = timestampRegex + eventRegex;
string tag = m_parser->m_regexList[i]["tag"];
vector<string> params = m_parser->m_regexList[i]["params"];
string tag = jsonList[i]["tag"];
vector<string> params = jsonList[i]["params"];
vector<string> timestampParams = { "month", "day", "time" };
params.insert(params.begin(), timestampParams.begin(), timestampParams.end());
m_parser->m_regexList[i]["params"] = params;
regex expr(regexString);
expression = expr;
} catch (domain_error& deException) {
parseParams(params, eventParams);
rs.params = eventParams;
rs.tag = tag;
rs.regexExpression = expression;
regexList.push_back(rs);
} catch (domain_error& deException) {
SWSS_LOG_ERROR("Missing required key, throws exception: %s\n", deException.what());
return false;
} catch (regex_error& reException) {
SWSS_LOG_ERROR("Invalid regex, throws exception: %s\n", reException.what());
return false;
}
m_parser->m_expressions.push_back(expression);
}

if(m_parser->m_expressions.empty()) {
if(regexList.empty()) {
SWSS_LOG_ERROR("Empty list of regex expressions.\n");
return false;
}

m_parser->m_regexList = regexList;

regexFile.close();
return true;
}
Expand Down
48 changes: 22 additions & 26 deletions src/sonic-eventd/rsyslog_plugin/syslog_parser.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,49 +14,45 @@
bool SyslogParser::parseMessage(string message, string& eventTag, event_params_t& paramMap, lua_State* luaState) {
for(long unsigned int i = 0; i < m_regexList.size(); i++) {
smatch matchResults;
vector<string> params = m_regexList[i]["params"];
if(!regex_search(message, matchResults, m_expressions[i]) || params.size() != matchResults.size() - 1 || matchResults.size() < 4) {
if(!regex_search(message, matchResults, m_regexList[i].regexExpression) || m_regexList[i].params.size() != matchResults.size() - 1 || matchResults.size() < 4) {
continue;
}

string formattedTimestamp;
if(!matchResults[1].str().empty() && !matchResults[2].str().empty() && !matchResults[3].str().empty()) { // found timestamp components
string formattedTimestamp = m_timestampFormatter->changeTimestampFormat({ matchResults[1].str(), matchResults[2].str(), matchResults[3].str() });
if(!formattedTimestamp.empty()) {
paramMap["timestamp"] = formattedTimestamp;
} else {
SWSS_LOG_ERROR("Timestamp is invalid and is not able to be formatted");
}
formattedTimestamp = m_timestampFormatter->changeTimestampFormat({ matchResults[1].str(), matchResults[2].str(), matchResults[3].str() });
}
if(!formattedTimestamp.empty()) {
paramMap["timestamp"] = formattedTimestamp;
} else {
SWSS_LOG_ERROR("Timestamp is invalid and is not able to be formatted");
}

// found matching regex
eventTag = m_regexList[i]["tag"];
eventTag = m_regexList[i].tag;
// check params for lua code
for(long unsigned int j = 3; j < params.size(); j++) {
auto delimPos = params[j].find(':');
for(long unsigned int j = 3; j < m_regexList[i].params.size(); j++) {
string resultValue = matchResults[j + 1].str();
if(delimPos == string::npos) { // no lua code
paramMap[params[j]] = resultValue;
string paramName = m_regexList[i].params[j].paramName;
const char* luaCode = m_regexList[i].params[j].luaCode.c_str();

if(luaCode == NULL || *luaCode == 0) {
SWSS_LOG_INFO("Invalid lua code, empty or missing");
paramMap[paramName] = resultValue;
continue;
}
// have to execute lua script
string param = params[j].substr(0, delimPos);
string luaString = params[j].substr(delimPos + 1);
if(luaString.empty()) { // empty lua code
SWSS_LOG_INFO("Lua code missing after :, skipping operation");
paramMap[param] = resultValue;
continue;
}
const char* luaCode = luaString.c_str();

// execute lua code
lua_pushstring(luaState, resultValue.c_str());
lua_setglobal(luaState, "arg");
if(luaL_dostring(luaState, luaCode) == 0) {
lua_pop(luaState, lua_gettop(luaState));
} else {
} else { // error in lua code
SWSS_LOG_ERROR("Invalid lua code, unable to do operation.\n");
paramMap[param] = resultValue;
paramMap[paramName] = resultValue;
continue;
}
lua_getglobal(luaState, "ret");
paramMap[param] = lua_tostring(luaState, -1);
paramMap[paramName] = lua_tostring(luaState, -1);
lua_pop(luaState, 1);
}
return true;
Expand Down
16 changes: 13 additions & 3 deletions src/sonic-eventd/rsyslog_plugin/syslog_parser.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,17 +18,27 @@ extern "C"
using namespace std;
using json = nlohmann::json;

struct EventParam {
string paramName;
string luaCode;
};

struct RegexStruct {
regex regexExpression;
vector<EventParam> params;
string tag;
};

/**
* Syslog Parser is responsible for parsing log messages fed by rsyslog.d and returns
* matched result to rsyslog_plugin to use with events publish API
*
*/

class SyslogParser {
public:
public:
unique_ptr<TimestampFormatter> m_timestampFormatter;
vector<regex> m_expressions;
json m_regexList = json::array();
vector<RegexStruct> m_regexList;
bool parseMessage(string message, string& tag, event_params_t& paramDict, lua_State* luaState);
SyslogParser();
};
Expand Down
108 changes: 62 additions & 46 deletions src/sonic-eventd/rsyslog_plugin_tests/rsyslog_plugin_ut.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,17 +19,30 @@ using namespace std;
using namespace swss;
using json = nlohmann::json;

vector<EventParam> createEventParams(vector<string> params, vector<string> luaCodes) {
vector<EventParam> eventParams;
for(long unsigned int i = 0; i < params.size(); i++) {
EventParam ep = EventParam();
ep.paramName = params[i];
ep.luaCode = luaCodes[i];
eventParams.push_back(ep);
}
return eventParams;
}

TEST(syslog_parser, matching_regex) {
json jList = json::array();
vector<regex> testExpressions;
vector<RegexStruct> regexList;
string regexString = "^([a-zA-Z]{3})?\\s*([0-9]{1,2})?\\s*([0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{0,6})?\\s*message (.*) other_data (.*) even_more_data (.*)";
json jTest;
jTest["tag"] = "test_tag";
jTest["regex"] = regexString;
jTest["params"] = { "month", "day", "time", "message", "other_data", "even_more_data" };
jList.push_back(jTest);
vector<string> params = { "month", "day", "time", "message", "other_data", "even_more_data" };
vector<string> luaCodes = { "", "", "", "", "", "" };
regex expression(regexString);
testExpressions.push_back(expression);

RegexStruct rs = RegexStruct();
rs.tag = "test_tag";
rs.regexExpression = expression;
rs.params = createEventParams(params, luaCodes);
regexList.push_back(rs);

string tag;
event_params_t paramDict;
Expand All @@ -40,8 +53,7 @@ TEST(syslog_parser, matching_regex) {
expectedDict["even_more_data"] = "test_data";

unique_ptr<SyslogParser> parser(new SyslogParser());
parser->m_expressions = testExpressions;
parser->m_regexList = jList;
parser->m_regexList = regexList;
lua_State* luaState = luaL_newstate();
luaL_openlibs(luaState);

Expand All @@ -55,15 +67,17 @@ TEST(syslog_parser, matching_regex) {

TEST(syslog_parser, matching_regex_timestamp) {
json jList = json::array();
vector<regex> testExpressions;
vector<RegexStruct> regexList;
string regexString = "^([a-zA-Z]{3})?\\s*([0-9]{1,2})?\\s*([0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{0,6})?\\s*message (.*) other_data (.*)";
json jTest;
jTest["tag"] = "test_tag";
jTest["regex"] = regexString;
jTest["params"] = { "month", "day", "time", "message", "other_data" };
jList.push_back(jTest);
vector<string> params = { "month", "day", "time", "message", "other_data" };
vector<string> luaCodes = { "", "", "", "", "" };
regex expression(regexString);
testExpressions.push_back(expression);

RegexStruct rs = RegexStruct();
rs.tag = "test_tag";
rs.regexExpression = expression;
rs.params = createEventParams(params, luaCodes);
regexList.push_back(rs);

string tag;
event_params_t paramDict;
Expand All @@ -74,8 +88,7 @@ TEST(syslog_parser, matching_regex_timestamp) {
expectedDict["timestamp"] = "2022-07-21T02:10:00.000000Z";

unique_ptr<SyslogParser> parser(new SyslogParser());
parser->m_expressions = testExpressions;
parser->m_regexList = jList;
parser->m_regexList = regexList;
lua_State* luaState = luaL_newstate();
luaL_openlibs(luaState);

Expand All @@ -89,22 +102,23 @@ TEST(syslog_parser, matching_regex_timestamp) {

TEST(syslog_parser, no_matching_regex) {
json jList = json::array();
vector<regex> testExpressions;
string regexString = "^([a-zA-Z]{3})?\\s*([0-9]{1,2})?\\s*([0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{0,6})?\s*no match";
json jTest;
jTest["tag"] = "test_tag";
jTest["regex"] = regexString;
jTest["params"] = { "month", "day", "time" };
jList.push_back(jTest);
vector<RegexStruct> regexList;
string regexString = "^([a-zA-Z]{3})?\\s*([0-9]{1,2})?\\s*([0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{0,6})?\\s*no match";
vector<string> params = { "month", "day", "time" };
vector<string> luaCodes = { "", "", "" };
regex expression(regexString);
testExpressions.push_back(expression);

RegexStruct rs = RegexStruct();
rs.tag = "test_tag";
rs.regexExpression = expression;
rs.params = createEventParams(params, luaCodes);
regexList.push_back(rs);

string tag;
event_params_t paramDict;

unique_ptr<SyslogParser> parser(new SyslogParser());
parser->m_expressions = testExpressions;
parser->m_regexList = jList;
parser->m_regexList = regexList;
lua_State* luaState = luaL_newstate();
luaL_openlibs(luaState);

Expand All @@ -116,15 +130,17 @@ TEST(syslog_parser, no_matching_regex) {

TEST(syslog_parser, lua_code_valid_1) {
json jList = json::array();
vector<regex> testExpressions;
vector<RegexStruct> regexList;
string regexString = "^([a-zA-Z]{3})?\\s*([0-9]{1,2})?\\s*([0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{0,6})?\\s*.* (sent|received) (?:to|from) .* ([0-9]{2,3}.[0-9]{2,3}.[0-9]{2,3}.[0-9]{2,3}) active ([1-9]{1,3})/([1-9]{1,3}) .*";
json jTest;
jTest["tag"] = "test_tag";
jTest["regex"] = regexString;
jTest["params"] = { "month", "day", "time", "is-sent:ret=tostring(arg==\"sent\")", "ip", "major-code", "minor-code" };
jList.push_back(jTest);
vector<string> params = { "month", "day", "time", "is-sent", "ip", "major-code", "minor-code" };
vector<string> luaCodes = { "", "", "", "ret=tostring(arg==\"sent\")", "", "", "" };
regex expression(regexString);
testExpressions.push_back(expression);

RegexStruct rs = RegexStruct();
rs.tag = "test_tag";
rs.regexExpression = expression;
rs.params = createEventParams(params, luaCodes);
regexList.push_back(rs);

string tag;
event_params_t paramDict;
Expand All @@ -136,8 +152,7 @@ TEST(syslog_parser, lua_code_valid_1) {
expectedDict["minor-code"] = "2";

unique_ptr<SyslogParser> parser(new SyslogParser());
parser->m_expressions = testExpressions;
parser->m_regexList = jList;
parser->m_regexList = regexList;
lua_State* luaState = luaL_newstate();
luaL_openlibs(luaState);

Expand All @@ -151,15 +166,17 @@ TEST(syslog_parser, lua_code_valid_1) {

TEST(syslog_parser, lua_code_valid_2) {
json jList = json::array();
vector<regex> testExpressions;
vector<RegexStruct> regexList;
string regexString = "([a-zA-Z]{3})?\\s*([0-9]{1,2})?\\s*([0-9]{2}:[0-9]{2}:[0-9]{2}.[0-9]{0,6})?\\s*.* (sent|received) (?:to|from) .* ([0-9]{2,3}.[0-9]{2,3}.[0-9]{2,3}.[0-9]{2,3}) active ([1-9]{1,3})/([1-9]{1,3}) .*";
json jTest;
jTest["tag"] = "test_tag";
jTest["regex"] = regexString;
jTest["params"] = { "month", "day", "time", "is-sent:ret=tostring(arg==\"sent\")", "ip", "major-code", "minor-code" };
jList.push_back(jTest);
vector<string> params = { "month", "day", "time", "is-sent", "ip", "major-code", "minor-code" };
vector<string> luaCodes = { "", "", "", "ret=tostring(arg==\"sent\")", "", "", "" };
regex expression(regexString);
testExpressions.push_back(expression);

RegexStruct rs = RegexStruct();
rs.tag = "test_tag";
rs.regexExpression = expression;
rs.params = createEventParams(params, luaCodes);
regexList.push_back(rs);

string tag;
event_params_t paramDict;
Expand All @@ -172,8 +189,7 @@ TEST(syslog_parser, lua_code_valid_2) {
expectedDict["timestamp"] = "2022-12-03T12:36:24.503424Z";

unique_ptr<SyslogParser> parser(new SyslogParser());
parser->m_expressions = testExpressions;
parser->m_regexList = jList;
parser->m_regexList = regexList;
lua_State* luaState = luaL_newstate();
luaL_openlibs(luaState);

Expand Down
Loading

0 comments on commit 2d09ad4

Please sign in to comment.