Structure for negative filter logstorage (#557)

Add 2 optional fields for excluded applications and contexts
Store value of these fields from config file



Filter message base on excluded applications and contexts

Logstorage does not support both multiple excluded applications and contexts

Check applications and contexts from message, if they existed in excluded list
the message will be not stored in logstorage buffer



Unit test update for negative filter

+ Add test suites for new functions
+ Add test cases for modification in current implementation
+ Update bash script for offline logstorage test



Update documentation to introduce negative filter

+ Add sequence diagram for negative filter
+ Add activity datagram for negative filter
+ Write description for negative filter in logstorage documentation
+ Write guideline for using negative filtern



Improvement for negative filter unit test

+ Add strncmp for checking success id storing process
+ Add comments for calloc functions to initialize config pointer



Initialize pointers in test suites

+ Add 3 more test scenarios for negative filter
+ Set unused excluded_apid/ctids fields to NULL

Signed-off-by: LUU QUANG MINH <Minh.LuuQuang@vn.bosch.com>
Co-authored-by: Le Tin <tin.le@vn.bosch.com>

doc/dlt_offline_logstorage.md

@@ -336,3 +336,56 @@ the clients are able to update any log level to user contexts.
 
 By setting ```MaintainLogstorageLogLevel=ON``` or ```MaintainLogstorageLogLevel=1```
 or not set, the logstorage will maintain its log level as the highest priority.
+
+## Logstorage Negative Filter
+
+The DLT Logstorage also provides negative filter feature, which allows filtering
+unwanted Application or Context.
+
+The following strategies are implemented:
+- Exclude a pair of Application and Context
+
+  ```
+  [FILTER1]
+  ...
+  ExcludedLogAppName=LOG
+  ExcludedContextName=TEST
+  ```
+- Exclude single Application or list of Applications
+
+  ```
+  [FILTER1]
+  ...
+  ExcludedLogAppName=LOG
+
+  [FILTER2]
+  ...
+  ExcludedLogAppName=LOG1,LOG2,LOG3
+  ```
+- Exclude single Context or list of Contexts
+
+  ```
+  [FILTER1]
+  ...
+  ExcludedContextName=TEST
+
+  [FILTER2]
+  ...
+  ExcludedContextName=TEST1,TEST2,TEST3
+  ```
+
+Note :
+
+DLT offline logstorage does not support multiple Application and multiple Context filter.
+The following configuration is not supported and the behavior will be undefinded:
+  ```
+  [FILTER1]
+  ...
+  LogAppName=LOG1,LOG2,LOG3
+  ContextName=TEST1,TEST2,TEST3
+
+  [FILTER2]
+  ...
+  ExcludedLogAppName=LOG1,LOG2,LOG3
+  ExcludedContextName=TEST1,TEST2,TEST3
+  ```

src/offlinelogstorage/dlt_offline_logstorage.c

@@ -56,6 +56,16 @@ DLT_STATIC void dlt_logstorage_filter_config_free(DltLogStorageFilterConfig *dat
         data->ctids = NULL;
     }
 
+    if (data->excluded_apids) {
+        free(data->excluded_apids);
+        data->excluded_apids = NULL;
+    }
+
+    if (data->excluded_ctids) {
+        free(data->excluded_ctids);
+        data->excluded_ctids = NULL;
+    }
+
     if (data->file_name) {
         free(data->file_name);
         data->file_name = NULL;
@@ -162,6 +172,12 @@ DLT_STATIC int dlt_logstorage_list_add_config(DltLogStorageFilterConfig *data,
     if (data->ctids != NULL)
         (*listdata)->ctids = strdup(data->ctids);
 
+    if (data->excluded_apids != NULL)
+        (*listdata)->excluded_apids = strdup(data->excluded_apids);
+
+    if (data->excluded_ctids != NULL)
+        (*listdata)->excluded_ctids = strdup(data->excluded_ctids);
+
     if (data->file_name != NULL)
         (*listdata)->file_name = strdup(data->file_name);
 
@@ -529,6 +545,45 @@ DLT_STATIC int dlt_logstorage_get_keys_list(char *ids, char *sep, char **list,
     return 0;
 }
 
+DLT_STATIC bool dlt_logstorage_check_excluded_ids(char *id, char *delim, char *excluded_ids)
+{
+    char *token = NULL;
+    char *tmp_token = NULL;
+    char *ids_local = NULL;
+
+    if ((id == NULL) || (delim == NULL) || (excluded_ids == NULL)) {
+        dlt_vlog(LOG_ERR, "%s: Invalid parameters\n", __func__);
+        return false;
+    }
+
+    ids_local = strdup(excluded_ids);
+
+    if (ids_local == NULL) {
+        dlt_vlog(LOG_ERR, "%s: Cannot duplicate string.\n", __func__);
+        return false;
+    }
+
+    token = strtok_r(ids_local, delim, &tmp_token);
+
+    if (token == NULL) {
+        dlt_vlog(LOG_ERR, "%s: %s could not be parsed.\n", __func__, ids_local);
+        free(ids_local);
+        return false;
+    }
+
+    while (token != NULL) {
+        if(strncmp(id, token, DLT_ID_SIZE) == 0) {
+            free(ids_local);
+            return true;
+        }
+
+        token = strtok_r(NULL, delim, &tmp_token);
+    }
+
+    free(ids_local);
+    return false;
+}
+
 /**
  * dlt_logstorage_create_keys_only_ctid
  *
@@ -969,6 +1024,28 @@ DLT_STATIC int dlt_logstorage_check_ctids(DltLogStorageFilterConfig *config,
     return dlt_logstorage_read_list_of_names(&config->ctids, (const char*)value);
 }
 
+DLT_STATIC int dlt_logstorage_store_config_excluded_apids(DltLogStorageFilterConfig *config,
+                                          char *value)
+{
+    if ((config == NULL) || (value == NULL)) {
+        dlt_vlog(LOG_ERR, "%s: Invalid parameters\n", __func__);
+        return -1;
+    }
+
+    return dlt_logstorage_read_list_of_names(&config->excluded_apids, value);
+}
+
+DLT_STATIC int dlt_logstorage_store_config_excluded_ctids(DltLogStorageFilterConfig *config,
+                                          char *value)
+{
+    if ((config == NULL) || (value == NULL)) {
+        dlt_vlog(LOG_ERR, "%s: Invalid parameters\n", __func__);
+        return -1;
+    }
+
+    return dlt_logstorage_read_list_of_names(&config->excluded_ctids, (const char*)value);
+}
+
 DLT_STATIC int dlt_logstorage_set_loglevel(int *log_level,
                                            int value)
 {
@@ -1327,6 +1404,16 @@ DLT_STATIC DltLogstorageFilterConf
         .func = dlt_logstorage_check_ctids,
         .is_opt = 1
     },
+    [DLT_LOGSTORAGE_FILTER_CONF_EXCLUDED_LOGAPPNAME] = {
+        .key = "ExcludedLogAppName",
+        .func = dlt_logstorage_store_config_excluded_apids,
+        .is_opt = 1
+    },
+    [DLT_LOGSTORAGE_FILTER_CONF_EXCLUDED_CONTEXTNAME] = {
+        .key = "ExcludedContextName",
+        .func = dlt_logstorage_store_config_excluded_ctids,
+        .is_opt = 1
+    },
     [DLT_LOGSTORAGE_FILTER_CONF_LOGLEVEL] = {
         .key = "LogLevel",
         .func = dlt_logstorage_check_loglevel,
@@ -1397,6 +1484,16 @@ DLT_STATIC DltLogstorageFilterConf
         .func = dlt_logstorage_check_ctids,
         .is_opt = 0
     },
+    [DLT_LOGSTORAGE_FILTER_CONF_EXCLUDED_LOGAPPNAME] = {
+        .key = NULL,
+        .func = dlt_logstorage_store_config_excluded_apids,
+        .is_opt = 1
+    },
+    [DLT_LOGSTORAGE_FILTER_CONF_EXCLUDED_CONTEXTNAME] = {
+        .key = NULL,
+        .func = dlt_logstorage_store_config_excluded_ctids,
+        .is_opt = 1
+    },
     [DLT_LOGSTORAGE_FILTER_CONF_LOGLEVEL] = {
         .key = NULL,
         .func = dlt_logstorage_check_loglevel,
@@ -1466,6 +1563,16 @@ DLT_STATIC DltLogstorageFilterConf
         .func = dlt_logstorage_check_ctids,
         .is_opt = 0
     },
+    [DLT_LOGSTORAGE_FILTER_CONF_EXCLUDED_LOGAPPNAME] = {
+        .key = NULL,
+        .func = dlt_logstorage_store_config_excluded_apids,
+        .is_opt = 1
+    },
+    [DLT_LOGSTORAGE_FILTER_CONF_EXCLUDED_CONTEXTNAME] = {
+        .key = NULL,
+        .func = dlt_logstorage_store_config_excluded_ctids,
+        .is_opt = 1
+    },
     [DLT_LOGSTORAGE_FILTER_CONF_LOGLEVEL] = {
         .key = "LogLevel",
         .func = dlt_logstorage_check_loglevel,
@@ -1685,6 +1792,16 @@ DLT_STATIC int dlt_daemon_offline_setup_filter_properties(DltLogStorage *handle,
                 tmp_data.ctids = NULL;
             }
 
+            if (tmp_data.excluded_apids != NULL) {
+                free(tmp_data.excluded_apids);
+                tmp_data.excluded_apids = NULL;
+            }
+
+            if (tmp_data.excluded_ctids != NULL) {
+                free(tmp_data.excluded_ctids);
+                tmp_data.excluded_ctids = NULL;
+            }
+
             if (tmp_data.file_name != NULL) {
                 free(tmp_data.file_name);
                 tmp_data.file_name = NULL;
@@ -1704,6 +1821,11 @@ DLT_STATIC int dlt_daemon_offline_setup_filter_properties(DltLogStorage *handle,
         }
     }
 
+    if(dlt_logstorage_count_ids(tmp_data.excluded_apids) > 1 && dlt_logstorage_count_ids(tmp_data.excluded_ctids) > 1) {
+        dlt_vlog(LOG_WARNING, "%s: Logstorage does not support both multiple excluded applications and contexts\n", __func__);
+        return DLT_OFFLINE_LOGSTORAGE_FILTER_ERROR;
+    }
+
     /* filter configuration is valid */
     ret = dlt_logstorage_setup_table(handle, &tmp_data);
 
@@ -2339,6 +2461,33 @@ DLT_STATIC int dlt_logstorage_filter(DltLogStorage *handle,
                          "%s: ECUID does not match (Requested=%s, config[%d]=%s). Set the config to NULL and continue the filter loop\n",
                          __func__, ecuid, i, config[i]->ecuid);
                 config[i] = NULL;
+                continue;
+            }
+        }
+
+        if(config[i]->excluded_apids != NULL && config[i]->excluded_ctids != NULL) {
+            /* Filter on excluded application and context */
+            if(apid != NULL && ctid != NULL && dlt_logstorage_check_excluded_ids(apid, ",", config[i]->excluded_apids)
+              && dlt_logstorage_check_excluded_ids(ctid, ",", config[i]->excluded_ctids)) {
+                dlt_vlog(LOG_DEBUG, "%s: %s matches with [%s] and %s matches with [%s]. Set the config to NULL and continue the filter loop\n",
+                __func__, apid, config[i]->excluded_apids, ctid, config[i]->excluded_ctids);
+                config[i] = NULL;
+            }
+        }
+        else if(config[i]->excluded_apids == NULL) {
+            /* Only filter on excluded contexts */
+            if(ctid != NULL && config[i]->excluded_ctids != NULL && dlt_logstorage_check_excluded_ids(ctid, ",", config[i]->excluded_ctids)) {
+                dlt_vlog(LOG_DEBUG, "%s: %s matches with [%s]. Set the config to NULL and continue the filter loop\n",
+                __func__, ctid, config[i]->excluded_ctids);
+                config[i] = NULL;
+            }
+        }
+        else if(config[i]->excluded_ctids == NULL) {
+            /* Only filter on excluded applications */
+            if(apid != NULL && config[i]->excluded_apids != NULL && dlt_logstorage_check_excluded_ids(apid, ",", config[i]->excluded_apids)) {
+                dlt_vlog(LOG_DEBUG, "%s: %s matches with [%s]. Set the config to NULL and continue the filter loop\n",
+                __func__, apid, config[i]->excluded_apids);
+                config[i] = NULL;
             }
         }
     }

src/offlinelogstorage/dlt_offline_logstorage.h

@@ -171,6 +171,8 @@ struct DltLogStorageFilterConfig
     /* filter section */
     char *apids;                    /* Application IDs configured for filter */
     char *ctids;                    /* Context IDs configured for filter */
+    char *excluded_apids;           /* Excluded Application IDs configured for filter */
+    char *excluded_ctids;           /* Excluded Context IDs configured for filter */
     int log_level;                  /* Log level number configured for filter */
     int reset_log_level;            /* reset Log level to be sent on disconnect */
     char *file_name;                /* File name for log storage configured for filter */
@@ -265,6 +267,8 @@ typedef struct {
 typedef enum {
     DLT_LOGSTORAGE_FILTER_CONF_LOGAPPNAME = 0,
     DLT_LOGSTORAGE_FILTER_CONF_CONTEXTNAME,
+    DLT_LOGSTORAGE_FILTER_CONF_EXCLUDED_LOGAPPNAME,
+    DLT_LOGSTORAGE_FILTER_CONF_EXCLUDED_CONTEXTNAME,
     DLT_LOGSTORAGE_FILTER_CONF_LOGLEVEL,
     DLT_LOGSTORAGE_FILTER_CONF_RESET_LOGLEVEL,
     DLT_LOGSTORAGE_FILTER_CONF_FILE,

src/offlinelogstorage/dlt_offline_logstorage_internal.h

@@ -76,6 +76,12 @@ DLT_STATIC int dlt_logstorage_check_apids(DltLogStorageFilterConfig *config, cha
 
 DLT_STATIC int dlt_logstorage_check_ctids(DltLogStorageFilterConfig *config, char *value);
 
+DLT_STATIC int dlt_logstorage_store_config_excluded_apids(DltLogStorageFilterConfig *config, char *value);
+
+DLT_STATIC int dlt_logstorage_store_config_excluded_ctids(DltLogStorageFilterConfig *config, char *value);
+
+DLT_STATIC bool dlt_logstorage_check_excluded_ids(char *id, char *delim, char *excluded_ids);
+
 DLT_STATIC int dlt_logstorage_check_loglevel(DltLogStorageFilterConfig *config, char *value);
 
 DLT_STATIC int dlt_logstorage_check_gzip_compression(DltLogStorageFilterConfig *config, char *value);