detect/datajson: introduce feature

This patch introduces a new keyword datajson that is similar
to dataset with a twist. Where dataset allows match from sets,
datajson allows the same but also adds JSON data to the alert
event. This data is comint from the set definition it self.
For example, an ipv4 set will look like:

  10.16.1.11,{"test": "success","context":3}

The syntax is value and json data separated by a comma.

The syntax of the keyword is the following:

  datajson:isset,src_ip,type ip,load src.lst,key src_ip;

Compare to dataset, it just have a supplementary option key
that is used to indicate in which subobject the JSON value
should be added.

The information is added in the even under the alert.extra
subobject:

  "alert": {
    "extra": {
      "src_ip": {
        "test": "success",
        "context": 3
      },

The main interest of the feature is to be able to contextualize
a match. For example, if you have an IOC source, you can do

 value1,{"actor":"APT28","Country":"FR"}
 value2,{"actor":"APT32","Country":"NL"}

This way, a single dataset is able to produce context to the
event where it was not possible before and multiple signatures
had to be used.

Ticket: OISF#7372

etc/schema.json

@@ -216,6 +216,11 @@
                 "xff": {
                     "type": "string"
                 },
+                "extra": {
+                    "type": "object",
+                    "additionalProperties": true,
+                    "description": "Extra data created by keywords such as datajson"
+                },
                 "metadata": {
                     "type": "object",
                     "properties": {

src/Makefile.am

@@ -50,6 +50,7 @@ noinst_HEADERS = \
 	datasets.h \
 	datasets-ipv4.h \
 	datasets-ipv6.h \
+	datasets-json.h \
 	datasets-md5.h \
 	datasets-reputation.h \
 	datasets-sha256.h \
@@ -102,6 +103,7 @@ noinst_HEADERS = \
 	detect-config.h \
 	detect-content.h \
 	detect-csum.h \
+	detect-datajson.h \
 	detect-datarep.h \
 	detect-dataset.h \
 	detect-dce-iface.h \
@@ -667,6 +669,7 @@ libsuricata_c_a_SOURCES = \
 	detect-config.c \
 	detect-content.c \
 	detect-csum.c \
+	detect-datajson.c \
 	detect-datarep.c \
 	detect-dataset.c \
 	detect-dce-iface.c \

src/datasets-ipv4.c

@@ -56,3 +56,37 @@ uint32_t IPv4Hash(uint32_t hash_seed, void *s)
 void IPv4Free(void *s)
 {
 }
+
+int IPv4JsonSet(void *dst, void *src)
+{
+    IPv4TypeJson *src_s = src;
+    IPv4TypeJson *dst_s = dst;
+    memcpy(dst_s->ipv4, src_s->ipv4, sizeof(dst_s->ipv4));
+    dst_s->json.value = src_s->json.value;
+    dst_s->json.len = src_s->json.len;
+
+    return 0;
+}
+
+bool IPv4JsonCompare(void *a, void *b)
+{
+    const IPv4TypeJson *as = a;
+    const IPv4TypeJson *bs = b;
+
+    return (memcmp(as->ipv4, bs->ipv4, sizeof(as->ipv4)) == 0);
+}
+
+uint32_t IPv4JsonHash(uint32_t hash_seed, void *s)
+{
+    const IPv4TypeJson *str = s;
+    return hashword((uint32_t *)str->ipv4, 1, hash_seed);
+}
+
+// data stays in hash
+void IPv4JsonFree(void *s)
+{
+    const IPv4TypeJson *as = s;
+    if (as->json.value) {
+        SCFree(as->json.value);
+    }
+}

src/datasets-ipv4.h

@@ -25,15 +25,26 @@
 #define SURICATA_DATASETS_IPV4_H
 
 #include "datasets-reputation.h"
+#include "datasets-json.h"
 
 typedef struct IPv4Type {
     uint8_t ipv4[4];
     DataRepType rep;
 } IPv4Type;
 
+typedef struct IPv4TypeJson {
+    uint8_t ipv4[4];
+    DataJsonType json;
+} IPv4TypeJson;
+
 int IPv4Set(void *dst, void *src);
 bool IPv4Compare(void *a, void *b);
 uint32_t IPv4Hash(uint32_t hash_seed, void *s);
 void IPv4Free(void *s);
 
+int IPv4JsonSet(void *dst, void *src);
+bool IPv4JsonCompare(void *a, void *b);
+uint32_t IPv4JsonHash(uint32_t hash_seed, void *s);
+void IPv4JsonFree(void *s);
+
 #endif /* SURICATA_DATASETS_IPV4_H */

src/datasets-ipv6.c

@@ -56,3 +56,36 @@ uint32_t IPv6Hash(uint32_t hash_seed, void *s)
 void IPv6Free(void *s)
 {
 }
+
+int IPv6JsonSet(void *dst, void *src)
+{
+    IPv6TypeJson *src_s = src;
+    IPv6TypeJson *dst_s = dst;
+    memcpy(dst_s->ipv6, src_s->ipv6, sizeof(dst_s->ipv6));
+    dst_s->json.value = src_s->json.value;
+    dst_s->json.len = src_s->json.len;
+
+    return 0;
+}
+
+bool IPv6JsonCompare(void *a, void *b)
+{
+    const IPv6TypeJson *as = a;
+    const IPv6TypeJson *bs = b;
+
+    return (memcmp(as->ipv6, bs->ipv6, sizeof(as->ipv6)) == 0);
+}
+
+uint32_t IPv6JsonHash(uint32_t hash_seed, void *s)
+{
+    const IPv6TypeJson *str = s;
+    return hashword((uint32_t *)str->ipv6, 4, hash_seed);
+}
+
+void IPv6JsonFree(void *s)
+{
+    const IPv6TypeJson *as = s;
+    if (as->json.value) {
+        SCFree(as->json.value);
+    }
+}

src/datasets-ipv6.h

@@ -31,9 +31,19 @@ typedef struct IPv6Type {
     DataRepType rep;
 } IPv6Type;
 
+typedef struct IPv6TypeJson {
+    uint8_t ipv6[16];
+    DataJsonType json;
+} IPv6TypeJson;
+
 int IPv6Set(void *dst, void *src);
 bool IPv6Compare(void *a, void *b);
 uint32_t IPv6Hash(uint32_t hash_seed, void *s);
 void IPv6Free(void *s);
 
+int IPv6JsonSet(void *dst, void *src);
+bool IPv6JsonCompare(void *a, void *b);
+uint32_t IPv6JsonHash(uint32_t hash_seed, void *s);
+void IPv6JsonFree(void *s);
+
 #endif /* __DATASETS_IPV4_H__ */

src/datasets-json.h

@@ -0,0 +1,39 @@
+/* Copyright (C) 2024 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Eric Leblond <el@stamus-networks.com>
+ */
+
+#ifndef __DATASETS_JSON_H__
+#define __DATASETS_JSON_H__
+
+#include <suricata-common.h>
+
+typedef struct DataJsonType {
+    char *value;
+    size_t len;
+} DataJsonType;
+
+typedef struct DataJsonResultType {
+    bool found;
+    DataJsonType json;
+} DataJsonResultType;
+
+#endif /* __DATASETS_JSON_H__ */

src/datasets-md5.c

@@ -57,3 +57,36 @@ uint32_t Md5StrHash(uint32_t hash_seed, void *s)
 void Md5StrFree(void *s)
 {
 }
+
+int Md5StrJsonSet(void *dst, void *src)
+{
+    Md5TypeJson *src_s = src;
+    Md5TypeJson *dst_s = dst;
+    memcpy(dst_s->md5, src_s->md5, sizeof(dst_s->md5));
+    dst_s->json.value = src_s->json.value;
+    dst_s->json.len = src_s->json.len;
+    return 0;
+}
+
+bool Md5StrJsonCompare(void *a, void *b)
+{
+    const Md5TypeJson *as = a;
+    const Md5TypeJson *bs = b;
+
+    return (memcmp(as->md5, bs->md5, sizeof(as->md5)) == 0);
+}
+
+uint32_t Md5StrJsonHash(uint32_t hash_seed, void *s)
+{
+    const Md5TypeJson *str = s;
+    return hashword((uint32_t *)str->md5, sizeof(str->md5) / 4, hash_seed);
+}
+
+// data stays in hash
+void Md5StrJsonFree(void *s)
+{
+    const Md5TypeJson *as = s;
+    if (as->json.value) {
+        SCFree(as->json.value);
+    }
+}

src/datasets-md5.h

@@ -25,15 +25,26 @@
 #define SURICATA_DATASETS_MD5_H
 
 #include "datasets-reputation.h"
+#include "datasets-json.h"
 
 typedef struct Md5Type {
     uint8_t md5[16];
     DataRepType rep;
 } Md5Type;
 
+typedef struct Md5TypeJson {
+    uint8_t md5[16];
+    DataJsonType json;
+} Md5TypeJson;
+
 int Md5StrSet(void *dst, void *src);
 bool Md5StrCompare(void *a, void *b);
 uint32_t Md5StrHash(uint32_t hash_seed, void *s);
 void Md5StrFree(void *s);
 
+int Md5StrJsonSet(void *dst, void *src);
+bool Md5StrJsonCompare(void *a, void *b);
+uint32_t Md5StrJsonHash(uint32_t hash_seed, void *s);
+void Md5StrJsonFree(void *s);
+
 #endif /* SURICATA_DATASETS_MD5_H */

src/datasets-sha256.c

@@ -56,3 +56,36 @@ void Sha256StrFree(void *s)
 {
     // no dynamic data
 }
+
+int Sha256StrJsonSet(void *dst, void *src)
+{
+    Sha256TypeJson *src_s = src;
+    Sha256TypeJson *dst_s = dst;
+    memcpy(dst_s->sha256, src_s->sha256, sizeof(dst_s->sha256));
+    dst_s->json.value = src_s->json.value;
+    dst_s->json.len = src_s->json.len;
+    return 0;
+}
+
+bool Sha256StrJsonCompare(void *a, void *b)
+{
+    Sha256TypeJson *as = a;
+    Sha256TypeJson *bs = b;
+
+    return (memcmp(as->sha256, bs->sha256, sizeof(as->sha256)) == 0);
+}
+
+uint32_t Sha256StrJsonHash(uint32_t hash_seed, void *s)
+{
+    Sha256TypeJson *str = s;
+    return hashword((uint32_t *)str->sha256, sizeof(str->sha256) / 4, hash_seed);
+}
+
+// data stays in hash
+void Sha256StrJsonFree(void *s)
+{
+    const Sha256TypeJson *as = s;
+    if (as->json.value) {
+        SCFree(as->json.value);
+    }
+}

src/datasets-sha256.h

@@ -25,15 +25,26 @@
 #define SURICATA_DATASETS_SHA256_H
 
 #include "datasets-reputation.h"
+#include "datasets-json.h"
 
 typedef struct Sha256Type {
     uint8_t sha256[32];
     DataRepType rep;
 } Sha256Type;
 
+typedef struct Sha256TypeJson {
+    uint8_t sha256[32];
+    DataJsonType json;
+} Sha256TypeJson;
+
 int Sha256StrSet(void *dst, void *src);
 bool Sha256StrCompare(void *a, void *b);
 uint32_t Sha256StrHash(uint32_t hash_seed, void *s);
 void Sha256StrFree(void *s);
 
+int Sha256StrJsonSet(void *dst, void *src);
+bool Sha256StrJsonCompare(void *a, void *b);
+uint32_t Sha256StrJsonHash(uint32_t hash_seed, void *s);
+void Sha256StrJsonFree(void *s);
+
 #endif /* SURICATA_DATASETS_SHA256_H */

src/datasets-string.c

@@ -103,3 +103,62 @@ void StringFree(void *s)
     StringType *str = s;
     SCFree(str->ptr);
 }
+
+int StringJsonAsBase64(const void *s, char *out, size_t out_size)
+{
+    const StringTypeJson *str = s;
+
+    unsigned long len = Base64EncodeBufferSize(str->len);
+    uint8_t encoded_data[len];
+    if (Base64Encode((unsigned char *)str->ptr, str->len, encoded_data, &len) != SC_BASE64_OK)
+        return 0;
+
+    strlcpy(out, (const char *)encoded_data, out_size);
+    strlcat(out, "\n", out_size);
+    return strlen(out);
+}
+
+int StringJsonSet(void *dst, void *src)
+{
+    StringTypeJson *src_s = src;
+    StringTypeJson *dst_s = dst;
+    SCLogDebug("dst %p src %p, src_s->ptr %p src_s->len %u", dst, src, src_s->ptr, src_s->len);
+
+    dst_s->len = src_s->len;
+    dst_s->ptr = SCMalloc(dst_s->len);
+    BUG_ON(dst_s->ptr == NULL);
+    memcpy(dst_s->ptr, src_s->ptr, dst_s->len);
+
+    dst_s->json.value = src_s->json.value;
+    dst_s->json.len = src_s->json.len;
+
+    SCLogDebug("dst %p src %p, dst_s->ptr %p dst_s->len %u", dst, src, dst_s->ptr, dst_s->len);
+    return 0;
+}
+
+bool StringJsonCompare(void *a, void *b)
+{
+    const StringTypeJson *as = a;
+    const StringTypeJson *bs = b;
+
+    if (as->len != bs->len)
+        return false;
+
+    return (memcmp(as->ptr, bs->ptr, as->len) == 0);
+}
+
+uint32_t StringJsonHash(uint32_t hash_seed, void *s)
+{
+    StringTypeJson *str = s;
+    return hashlittle_safe(str->ptr, str->len, hash_seed);
+}
+
+// base data stays in hash
+void StringJsonFree(void *s)
+{
+    StringTypeJson *str = s;
+    SCFree(str->ptr);
+    if (str->json.value) {
+        SCFree(str->json.value);
+    }
+}