From 2ac6a02fd2724a3cce208c175104363df0119694 Mon Sep 17 00:00:00 2001
From: Jeff Lucovsky <jlucovsky@oisf.net>
Date: Thu, 12 Sep 2024 10:44:05 -0400
Subject: [PATCH 1/6] transforms: Make ctx available

This commit makes the detection engine thread context available for
transforms to use. The Lua transform requires this value.

Issue: 2290
---
 src/detect-dns-answer-name.c    |  4 ++--
 src/detect-dns-query-name.c     |  4 ++--
 src/detect-dns-query.c          |  4 ++--
 src/detect-engine-frame.c       | 18 +++++++++---------
 src/detect-engine-helper.c      |  4 ++--
 src/detect-engine.c             |  9 ++++++---
 src/detect-engine.h             |  6 +++---
 src/detect-file-data.c          |  7 ++++---
 src/detect-filemagic.c          |  4 ++--
 src/detect-filename.c           |  2 +-
 src/detect-http-header.c        | 12 ++++++------
 src/detect-ike-vendor.c         |  4 ++--
 src/detect-krb5-cname.c         |  6 +++---
 src/detect-krb5-sname.c         |  6 +++---
 src/detect-quic-cyu-hash.c      |  4 ++--
 src/detect-quic-cyu-string.c    |  4 ++--
 src/detect-tls-alpn.c           |  6 +++---
 src/detect-tls-certs.c          |  6 +++---
 src/detect-tls-subjectaltname.c |  2 +-
 src/detect.h                    |  1 +
 20 files changed, 59 insertions(+), 54 deletions(-)

diff --git a/src/detect-dns-answer-name.c b/src/detect-dns-answer-name.c
index dc1272d47510..91ebb29c8bea 100644
--- a/src/detect-dns-answer-name.c
+++ b/src/detect-dns-answer-name.c
@@ -61,10 +61,10 @@ static InspectionBuffer *GetBuffer(DetectEngineThreadCtx *det_ctx,
     uint32_t data_len = 0;
 
     if (!SCDnsTxGetAnswerName(txv, to_client, index, &data, &data_len)) {
-        InspectionBufferSetupMultiEmpty(buffer);
+        InspectionBufferSetupMultiEmpty(det_ctx, buffer);
         return NULL;
     }
-    InspectionBufferSetupMulti(buffer, transforms, data, data_len);
+    InspectionBufferSetupMulti(det_ctx, buffer, transforms, data, data_len);
     buffer->flags = DETECT_CI_FLAGS_SINGLE;
     return buffer;
 }
diff --git a/src/detect-dns-query-name.c b/src/detect-dns-query-name.c
index ca1cc79fa4bf..3cad5b1740e2 100644
--- a/src/detect-dns-query-name.c
+++ b/src/detect-dns-query-name.c
@@ -61,10 +61,10 @@ static InspectionBuffer *GetBuffer(DetectEngineThreadCtx *det_ctx,
     uint32_t data_len = 0;
 
     if (!SCDnsTxGetQueryName(txv, to_client, index, &data, &data_len)) {
-        InspectionBufferSetupMultiEmpty(buffer);
+        InspectionBufferSetupMultiEmpty(det_ctx, buffer);
         return NULL;
     }
-    InspectionBufferSetupMulti(buffer, transforms, data, data_len);
+    InspectionBufferSetupMulti(det_ctx, buffer, transforms, data, data_len);
     buffer->flags = DETECT_CI_FLAGS_SINGLE;
     return buffer;
 }
diff --git a/src/detect-dns-query.c b/src/detect-dns-query.c
index ef510f15287a..f709b8871e22 100644
--- a/src/detect-dns-query.c
+++ b/src/detect-dns-query.c
@@ -82,10 +82,10 @@ static InspectionBuffer *DnsQueryGetData(DetectEngineThreadCtx *det_ctx,
     const uint8_t *data;
     uint32_t data_len;
     if (SCDnsTxGetQueryName(txv, false, local_id, &data, &data_len) == 0) {
-        InspectionBufferSetupMultiEmpty(buffer);
+        InspectionBufferSetupMultiEmpty(det_ctx, buffer);
         return NULL;
     }
-    InspectionBufferSetupMulti(buffer, transforms, data, data_len);
+    InspectionBufferSetupMulti(det_ctx, buffer, transforms, data, data_len);
     buffer->flags = DETECT_CI_FLAGS_SINGLE;
 
     SCReturnPtr(buffer, "InspectionBuffer");
diff --git a/src/detect-engine-frame.c b/src/detect-engine-frame.c
index fd3163d59732..1ba62cf1ded5 100644
--- a/src/detect-engine-frame.c
+++ b/src/detect-engine-frame.c
@@ -67,8 +67,8 @@ static bool SetupStreamCallbackData(struct FrameStreamData *dst, const TcpSessio
 
 static bool BufferSetup(struct FrameStreamData *fsd, InspectionBuffer *buffer, const uint8_t *input,
         const uint32_t input_len, const uint64_t input_offset);
-static void BufferSetupUdp(InspectionBuffer *buffer, const Frame *frame, const Packet *p,
-        const DetectEngineTransforms *transforms);
+static void BufferSetupUdp(DetectEngineThreadCtx *det_ctx, InspectionBuffer *buffer,
+        const Frame *frame, const Packet *p, const DetectEngineTransforms *transforms);
 
 void DetectRunPrefilterFrame(DetectEngineThreadCtx *det_ctx, const SigGroupHead *sgh, Packet *p,
         const Frames *frames, const Frame *frame, const AppProto alproto)
@@ -159,7 +159,7 @@ static void PrefilterMpmFrame(DetectEngineThreadCtx *det_ctx, const void *pectx,
         if (frame->offset >= p->payload_len)
             return;
 
-        BufferSetupUdp(buffer, frame, p, ctx->transforms);
+        BufferSetupUdp(det_ctx, buffer, frame, p, ctx->transforms);
         const uint32_t data_len = buffer->inspect_len;
         const uint8_t *data = buffer->inspect;
 
@@ -251,8 +251,8 @@ bool DetectRunFrameInspectRule(ThreadVars *tv, DetectEngineThreadCtx *det_ctx, c
     return false;
 }
 
-static void BufferSetupUdp(InspectionBuffer *buffer, const Frame *frame, const Packet *p,
-        const DetectEngineTransforms *transforms)
+static void BufferSetupUdp(DetectEngineThreadCtx *det_ctx, InspectionBuffer *buffer,
+        const Frame *frame, const Packet *p, const DetectEngineTransforms *transforms)
 {
     uint8_t ci_flags = DETECT_CI_FLAGS_START;
     uint32_t frame_len;
@@ -275,7 +275,7 @@ static void BufferSetupUdp(InspectionBuffer *buffer, const Frame *frame, const P
             AppLayerParserGetFrameNameById(p->flow->proto, p->flow->alproto, frame->type),
             frame->offset, frame->type, frame->len);
 
-    InspectionBufferSetupMulti(buffer, transforms, data, data_len);
+    InspectionBufferSetupMulti(det_ctx, buffer, transforms, data, data_len);
     buffer->inspect_offset = 0;
     buffer->flags = ci_flags;
 }
@@ -301,7 +301,7 @@ static int DetectFrameInspectUdp(DetectEngineThreadCtx *det_ctx,
         return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
 
     if (!buffer->initialized)
-        BufferSetupUdp(buffer, frame, p, transforms);
+        BufferSetupUdp(det_ctx, buffer, frame, p, transforms);
     DEBUG_VALIDATE_BUG_ON(!buffer->initialized);
     if (buffer->inspect == NULL)
         return DETECT_ENGINE_INSPECT_SIG_NO_MATCH;
@@ -361,7 +361,7 @@ static bool BufferSetup(struct FrameStreamData *fsd, InspectionBuffer *buffer, c
             if (fo_inspect_offset >= (uint64_t)frame->len) {
                 SCLogDebug("data entirely past frame (%" PRIu64 " > %" PRIi64 ")",
                         fo_inspect_offset, frame->len);
-                InspectionBufferSetupMultiEmpty(buffer);
+                InspectionBufferSetupMultiEmpty(fsd->det_ctx, buffer);
                 return false;
             }
 
@@ -387,7 +387,7 @@ static bool BufferSetup(struct FrameStreamData *fsd, InspectionBuffer *buffer, c
     }
     // PrintRawDataFp(stdout, data, data_len);
     SCLogDebug("fsd->transforms %p", fsd->transforms);
-    InspectionBufferSetupMulti(buffer, fsd->transforms, data, data_len);
+    InspectionBufferSetupMulti(fsd->det_ctx, buffer, fsd->transforms, data, data_len);
     SCLogDebug("inspect_offset %" PRIu64, fo_inspect_offset);
     buffer->inspect_offset = fo_inspect_offset;
     buffer->flags = ci_flags;
diff --git a/src/detect-engine-helper.c b/src/detect-engine-helper.c
index 740fec7618c8..2a1e96a75974 100644
--- a/src/detect-engine-helper.c
+++ b/src/detect-engine-helper.c
@@ -137,10 +137,10 @@ InspectionBuffer *DetectHelperGetMultiData(struct DetectEngineThreadCtx_ *det_ct
     uint32_t data_len = 0;
 
     if (!GetBuf(txv, flow_flags, index, &data, &data_len)) {
-        InspectionBufferSetupMultiEmpty(buffer);
+        InspectionBufferSetupMultiEmpty(det_ctx, buffer);
         return NULL;
     }
-    InspectionBufferSetupMulti(buffer, transforms, data, data_len);
+    InspectionBufferSetupMulti(det_ctx, buffer, transforms, data, data_len);
     buffer->flags = DETECT_CI_FLAGS_SINGLE;
     return buffer;
 }
diff --git a/src/detect-engine.c b/src/detect-engine.c
index 58b5c9967c8b..f70d59656026 100644
--- a/src/detect-engine.c
+++ b/src/detect-engine.c
@@ -1562,7 +1562,7 @@ void InspectionBufferInit(InspectionBuffer *buffer, uint32_t initial_size)
 }
 
 /** \brief setup the buffer empty */
-void InspectionBufferSetupMultiEmpty(InspectionBuffer *buffer)
+void InspectionBufferSetupMultiEmpty(DetectEngineThreadCtx *det_ctx, InspectionBuffer *buffer)
 {
 #ifdef DEBUG_VALIDATION
     DEBUG_VALIDATE_BUG_ON(buffer->initialized);
@@ -1572,11 +1572,12 @@ void InspectionBufferSetupMultiEmpty(InspectionBuffer *buffer)
     buffer->inspect_len = 0;
     buffer->len = 0;
     buffer->initialized = true;
+    buffer->det_ctx = det_ctx;
 }
 
 /** \brief setup the buffer with our initial data */
-void InspectionBufferSetupMulti(InspectionBuffer *buffer, const DetectEngineTransforms *transforms,
-        const uint8_t *data, const uint32_t data_len)
+void InspectionBufferSetupMulti(DetectEngineThreadCtx *det_ctx, InspectionBuffer *buffer,
+        const DetectEngineTransforms *transforms, const uint8_t *data, const uint32_t data_len)
 {
 #ifdef DEBUG_VALIDATION
     DEBUG_VALIDATE_BUG_ON(!buffer->multi);
@@ -1585,6 +1586,7 @@ void InspectionBufferSetupMulti(InspectionBuffer *buffer, const DetectEngineTran
     buffer->inspect_len = buffer->orig_len = data_len;
     buffer->len = 0;
     buffer->initialized = true;
+    buffer->det_ctx = det_ctx;
 
     InspectionBufferApplyTransforms(buffer, transforms);
 }
@@ -1603,6 +1605,7 @@ void InspectionBufferSetup(DetectEngineThreadCtx *det_ctx, const int list_id,
 #endif
             det_ctx->inspect.to_clear_queue[det_ctx->inspect.to_clear_idx++] = list_id;
     }
+    buffer->det_ctx = det_ctx;
     buffer->inspect = buffer->orig = data;
     buffer->inspect_len = buffer->orig_len = data_len;
     buffer->len = 0;
diff --git a/src/detect-engine.h b/src/detect-engine.h
index b0e443e1270a..6f1f61fe851c 100644
--- a/src/detect-engine.h
+++ b/src/detect-engine.h
@@ -37,9 +37,9 @@ void InspectionBufferApplyTransforms(InspectionBuffer *buffer,
         const DetectEngineTransforms *transforms);
 void InspectionBufferClean(DetectEngineThreadCtx *det_ctx);
 InspectionBuffer *InspectionBufferGet(DetectEngineThreadCtx *det_ctx, const int list_id);
-void InspectionBufferSetupMultiEmpty(InspectionBuffer *buffer);
-void InspectionBufferSetupMulti(InspectionBuffer *buffer, const DetectEngineTransforms *transforms,
-        const uint8_t *data, const uint32_t data_len);
+void InspectionBufferSetupMultiEmpty(DetectEngineThreadCtx *det_ctx, InspectionBuffer *buffer);
+void InspectionBufferSetupMulti(DetectEngineThreadCtx *det_ctx, InspectionBuffer *buffer,
+        const DetectEngineTransforms *transforms, const uint8_t *data, const uint32_t data_len);
 InspectionBuffer *InspectionBufferMultipleForListGet(
         DetectEngineThreadCtx *det_ctx, const int list_id, uint32_t local_id);
 
diff --git a/src/detect-file-data.c b/src/detect-file-data.c
index a721c08c7cf9..ca0a15e57db2 100644
--- a/src/detect-file-data.c
+++ b/src/detect-file-data.c
@@ -192,7 +192,8 @@ static inline InspectionBuffer *FiledataWithXformsGetDataCallback(DetectEngineTh
         return buffer;
     }
 
-    InspectionBufferSetupMulti(buffer, transforms, base_buffer->inspect, base_buffer->inspect_len);
+    InspectionBufferSetupMulti(
+            det_ctx, buffer, transforms, base_buffer->inspect, base_buffer->inspect_len);
     buffer->inspect_offset = base_buffer->inspect_offset;
     SCLogDebug("xformed buffer %p size %u", buffer, buffer->inspect_len);
     SCReturnPtr(buffer, "InspectionBuffer");
@@ -351,7 +352,7 @@ static InspectionBuffer *FiledataGetDataCallback(DetectEngineThreadCtx *det_ctx,
         SCLogDebug("content inspected: %" PRIu64, cur_file->content_inspected);
     }
 
-    InspectionBufferSetupMulti(buffer, NULL, data, data_len);
+    InspectionBufferSetupMulti(det_ctx, buffer, NULL, data, data_len);
     SCLogDebug("[list %d] [before] buffer offset %" PRIu64 "; buffer len %" PRIu32
                "; data_len %" PRIu32 "; file_size %" PRIu64,
             list_id, buffer->inspect_offset, buffer->inspect_len, data_len, file_size);
@@ -385,7 +386,7 @@ static InspectionBuffer *FiledataGetDataCallback(DetectEngineThreadCtx *det_ctx,
     SCReturnPtr(buffer, "InspectionBuffer");
 
 empty_return:
-    InspectionBufferSetupMultiEmpty(buffer);
+    InspectionBufferSetupMultiEmpty(det_ctx, buffer);
     return NULL;
 }
 
diff --git a/src/detect-filemagic.c b/src/detect-filemagic.c
index f23434d8666e..58519238d91a 100644
--- a/src/detect-filemagic.c
+++ b/src/detect-filemagic.c
@@ -278,7 +278,7 @@ static InspectionBuffer *FilemagicGetDataCallback(DetectEngineThreadCtx *det_ctx
                 (DetectFilemagicThreadData *)DetectThreadCtxGetKeywordThreadCtx(
                         det_ctx, det_ctx->de_ctx->filemagic_thread_ctx_id);
         if (tfilemagic == NULL) {
-            InspectionBufferSetupMultiEmpty(buffer);
+            InspectionBufferSetupMultiEmpty(det_ctx, buffer);
             return NULL;
         }
 
@@ -291,7 +291,7 @@ static InspectionBuffer *FilemagicGetDataCallback(DetectEngineThreadCtx *det_ctx
     const uint8_t *data = (const uint8_t *)cur_file->magic;
     uint32_t data_len = (uint32_t)strlen(cur_file->magic);
 
-    InspectionBufferSetupMulti(buffer, transforms, data, data_len);
+    InspectionBufferSetupMulti(det_ctx, buffer, transforms, data, data_len);
 
     SCReturnPtr(buffer, "InspectionBuffer");
 }
diff --git a/src/detect-filename.c b/src/detect-filename.c
index f75fdbd680fe..11e576071055 100644
--- a/src/detect-filename.c
+++ b/src/detect-filename.c
@@ -228,7 +228,7 @@ static InspectionBuffer *FilenameGetDataCallback(DetectEngineThreadCtx *det_ctx,
     const uint8_t *data = cur_file->name;
     uint32_t data_len = cur_file->name_len;
 
-    InspectionBufferSetupMulti(buffer, transforms, data, data_len);
+    InspectionBufferSetupMulti(det_ctx, buffer, transforms, data, data_len);
 
     SCReturnPtr(buffer, "InspectionBuffer");
 }
diff --git a/src/detect-http-header.c b/src/detect-http-header.c
index be825e5ec714..d4ad2890f621 100644
--- a/src/detect-http-header.c
+++ b/src/detect-http-header.c
@@ -519,15 +519,15 @@ static InspectionBuffer *GetHttp2HeaderData(DetectEngineThreadCtx *det_ctx,
     const uint8_t *b = NULL;
 
     if (rs_http2_tx_get_header(txv, flags, local_id, &b, &b_len) != 1) {
-        InspectionBufferSetupMultiEmpty(buffer);
+        InspectionBufferSetupMultiEmpty(det_ctx, buffer);
         return NULL;
     }
     if (b == NULL || b_len == 0) {
-        InspectionBufferSetupMultiEmpty(buffer);
+        InspectionBufferSetupMultiEmpty(det_ctx, buffer);
         return NULL;
     }
 
-    InspectionBufferSetupMulti(buffer, transforms, b, b_len);
+    InspectionBufferSetupMulti(det_ctx, buffer, transforms, b, b_len);
     buffer->flags = DETECT_CI_FLAGS_SINGLE;
 
     SCReturnPtr(buffer, "InspectionBuffer");
@@ -605,12 +605,12 @@ static InspectionBuffer *GetHttp1HeaderData(DetectEngineThreadCtx *det_ctx,
     // hdr_td->len is the number of header buffers
     if (local_id < hdr_td->len) {
         // we have one valid header buffer
-        InspectionBufferSetupMulti(
-                buffer, transforms, hdr_td->items[local_id].buffer, hdr_td->items[local_id].len);
+        InspectionBufferSetupMulti(det_ctx, buffer, transforms, hdr_td->items[local_id].buffer,
+                hdr_td->items[local_id].len);
         buffer->flags = DETECT_CI_FLAGS_SINGLE;
         SCReturnPtr(buffer, "InspectionBuffer");
     } // else there are no more header buffer to get
-    InspectionBufferSetupMultiEmpty(buffer);
+    InspectionBufferSetupMultiEmpty(det_ctx, buffer);
     return NULL;
 }
 
diff --git a/src/detect-ike-vendor.c b/src/detect-ike-vendor.c
index e3c09e9a44c6..7eaa82f32635 100644
--- a/src/detect-ike-vendor.c
+++ b/src/detect-ike-vendor.c
@@ -54,11 +54,11 @@ static InspectionBuffer *IkeVendorGetData(DetectEngineThreadCtx *det_ctx,
     const uint8_t *data;
     uint32_t data_len;
     if (rs_ike_tx_get_vendor(txv, local_id, &data, &data_len) == 0) {
-        InspectionBufferSetupMultiEmpty(buffer);
+        InspectionBufferSetupMultiEmpty(det_ctx, buffer);
         return NULL;
     }
 
-    InspectionBufferSetupMulti(buffer, transforms, data, data_len);
+    InspectionBufferSetupMulti(det_ctx, buffer, transforms, data, data_len);
     buffer->flags = DETECT_CI_FLAGS_SINGLE;
 
     SCReturnPtr(buffer, "InspectionBuffer");
diff --git a/src/detect-krb5-cname.c b/src/detect-krb5-cname.c
index 1411f7380806..5272b3643759 100644
--- a/src/detect-krb5-cname.c
+++ b/src/detect-krb5-cname.c
@@ -65,15 +65,15 @@ static InspectionBuffer *GetKrb5CNameData(DetectEngineThreadCtx *det_ctx,
     const uint8_t *b = NULL;
 
     if (rs_krb5_tx_get_cname(txv, local_id, &b, &b_len) != 1) {
-        InspectionBufferSetupMultiEmpty(buffer);
+        InspectionBufferSetupMultiEmpty(det_ctx, buffer);
         return NULL;
     }
     if (b == NULL || b_len == 0) {
-        InspectionBufferSetupMultiEmpty(buffer);
+        InspectionBufferSetupMultiEmpty(det_ctx, buffer);
         return NULL;
     }
 
-    InspectionBufferSetupMulti(buffer, transforms, b, b_len);
+    InspectionBufferSetupMulti(det_ctx, buffer, transforms, b, b_len);
     buffer->flags = DETECT_CI_FLAGS_SINGLE;
 
     SCReturnPtr(buffer, "InspectionBuffer");
diff --git a/src/detect-krb5-sname.c b/src/detect-krb5-sname.c
index 3cd6f0e222cc..06418ebebaa1 100644
--- a/src/detect-krb5-sname.c
+++ b/src/detect-krb5-sname.c
@@ -65,15 +65,15 @@ static InspectionBuffer *GetKrb5SNameData(DetectEngineThreadCtx *det_ctx,
     const uint8_t *b = NULL;
 
     if (rs_krb5_tx_get_sname(txv, local_id, &b, &b_len) != 1) {
-        InspectionBufferSetupMultiEmpty(buffer);
+        InspectionBufferSetupMultiEmpty(det_ctx, buffer);
         return NULL;
     }
     if (b == NULL || b_len == 0) {
-        InspectionBufferSetupMultiEmpty(buffer);
+        InspectionBufferSetupMultiEmpty(det_ctx, buffer);
         return NULL;
     }
 
-    InspectionBufferSetupMulti(buffer, transforms, b, b_len);
+    InspectionBufferSetupMulti(det_ctx, buffer, transforms, b, b_len);
     buffer->flags = DETECT_CI_FLAGS_SINGLE;
 
     SCReturnPtr(buffer, "InspectionBuffer");
diff --git a/src/detect-quic-cyu-hash.c b/src/detect-quic-cyu-hash.c
index 17836d1596b5..12874389283a 100644
--- a/src/detect-quic-cyu-hash.c
+++ b/src/detect-quic-cyu-hash.c
@@ -72,11 +72,11 @@ static InspectionBuffer *QuicHashGetData(DetectEngineThreadCtx *det_ctx,
     const uint8_t *data;
     uint32_t data_len;
     if (rs_quic_tx_get_cyu_hash(txv, local_id, &data, &data_len) == 0) {
-        InspectionBufferSetupMultiEmpty(buffer);
+        InspectionBufferSetupMultiEmpty(det_ctx, buffer);
         return NULL;
     }
 
-    InspectionBufferSetupMulti(buffer, transforms, data, data_len);
+    InspectionBufferSetupMulti(det_ctx, buffer, transforms, data, data_len);
     buffer->flags = DETECT_CI_FLAGS_SINGLE;
 
     SCReturnPtr(buffer, "InspectionBuffer");
diff --git a/src/detect-quic-cyu-string.c b/src/detect-quic-cyu-string.c
index c2460f11546c..8bf85b25e161 100644
--- a/src/detect-quic-cyu-string.c
+++ b/src/detect-quic-cyu-string.c
@@ -68,11 +68,11 @@ static InspectionBuffer *QuicStringGetData(DetectEngineThreadCtx *det_ctx,
     const uint8_t *data;
     uint32_t data_len;
     if (rs_quic_tx_get_cyu_string(txv, local_id, &data, &data_len) == 0) {
-        InspectionBufferSetupMultiEmpty(buffer);
+        InspectionBufferSetupMultiEmpty(det_ctx, buffer);
         return NULL;
     }
 
-    InspectionBufferSetupMulti(buffer, transforms, data, data_len);
+    InspectionBufferSetupMulti(det_ctx, buffer, transforms, data, data_len);
     buffer->flags = DETECT_CI_FLAGS_SINGLE;
 
     SCReturnPtr(buffer, "InspectionBuffer");
diff --git a/src/detect-tls-alpn.c b/src/detect-tls-alpn.c
index b4aa82f9c52a..5e38be662a59 100644
--- a/src/detect-tls-alpn.c
+++ b/src/detect-tls-alpn.c
@@ -122,7 +122,7 @@ static InspectionBuffer *TlsAlpnGetData(DetectEngineThreadCtx *det_ctx,
     }
 
     if (TAILQ_EMPTY(&connp->alpns)) {
-        InspectionBufferSetupMultiEmpty(buffer);
+        InspectionBufferSetupMultiEmpty(det_ctx, buffer);
         return NULL;
     }
 
@@ -137,11 +137,11 @@ static InspectionBuffer *TlsAlpnGetData(DetectEngineThreadCtx *det_ctx,
         }
     }
     if (a == NULL) {
-        InspectionBufferSetupMultiEmpty(buffer);
+        InspectionBufferSetupMultiEmpty(det_ctx, buffer);
         return NULL;
     }
 
-    InspectionBufferSetupMulti(buffer, transforms, a->alpn, a->size);
+    InspectionBufferSetupMulti(det_ctx, buffer, transforms, a->alpn, a->size);
     buffer->flags = DETECT_CI_FLAGS_SINGLE;
 
     SCReturnPtr(buffer, "InspectionBuffer");
diff --git a/src/detect-tls-certs.c b/src/detect-tls-certs.c
index f34c5e23bfb6..02bff2b1c26c 100644
--- a/src/detect-tls-certs.c
+++ b/src/detect-tls-certs.c
@@ -82,7 +82,7 @@ static InspectionBuffer *TlsCertsGetData(DetectEngineThreadCtx *det_ctx,
     }
 
     if (TAILQ_EMPTY(&connp->certs)) {
-        InspectionBufferSetupMultiEmpty(buffer);
+        InspectionBufferSetupMultiEmpty(det_ctx, buffer);
         return NULL;
     }
 
@@ -97,11 +97,11 @@ static InspectionBuffer *TlsCertsGetData(DetectEngineThreadCtx *det_ctx,
         }
     }
     if (cert == NULL) {
-        InspectionBufferSetupMultiEmpty(buffer);
+        InspectionBufferSetupMultiEmpty(det_ctx, buffer);
         return NULL;
     }
 
-    InspectionBufferSetupMulti(buffer, transforms, cert->cert_data, cert->cert_len);
+    InspectionBufferSetupMulti(det_ctx, buffer, transforms, cert->cert_data, cert->cert_len);
     buffer->flags = DETECT_CI_FLAGS_SINGLE;
 
     SCReturnPtr(buffer, "InspectionBuffer");
diff --git a/src/detect-tls-subjectaltname.c b/src/detect-tls-subjectaltname.c
index 350db5d6f655..397a5e67cfaa 100644
--- a/src/detect-tls-subjectaltname.c
+++ b/src/detect-tls-subjectaltname.c
@@ -121,7 +121,7 @@ static InspectionBuffer *TlsSubjectAltNameGetData(DetectEngineThreadCtx *det_ctx
         return NULL;
     }
 
-    InspectionBufferSetupMulti(buffer, transforms, (const uint8_t *)connp->cert0_sans[idx],
+    InspectionBufferSetupMulti(det_ctx, buffer, transforms, (const uint8_t *)connp->cert0_sans[idx],
             strlen(connp->cert0_sans[idx]));
     buffer->flags = DETECT_CI_FLAGS_SINGLE;
 
diff --git a/src/detect.h b/src/detect.h
index cf8f4335b197..ae615be7a7f2 100644
--- a/src/detect.h
+++ b/src/detect.h
@@ -371,6 +371,7 @@ struct DetectEngineThreadCtx_;// DetectEngineThreadCtx;
  * Prefilter and inspection will only deal with 'inspect'. */
 
 typedef struct InspectionBuffer {
+    struct DetectEngineThreadCtx_ *det_ctx;
     const uint8_t *inspect; /**< active pointer, points either to ::buf or ::orig */
     uint64_t inspect_offset;
     uint32_t inspect_len;   /**< size of active data. See to ::len or ::orig_len */

From c223c30c7ee35dc75b66d14cf1612d75ddf058ba Mon Sep 17 00:00:00 2001
From: Jeff Lucovsky <jlucovsky@oisf.net>
Date: Thu, 12 Sep 2024 10:47:26 -0400
Subject: [PATCH 2/6] hash: Provide detection engine context to free fn

Issue: 2290

This commit extends the hash table logic with an alternate free function
that provides the detection engine context.

Users that wish to use the next functionality must use the
HashListTableInitWithCtx function when initializing the hash table.
Using this interface will result in the hash table "free with context"
function (new) being used instead.
---
 src/detect-engine.c | 18 ++++++-----
 src/util-hashlist.c | 78 +++++++++++++++++++++++++++++++++++++++++++++
 src/util-hashlist.h |  6 ++++
 3 files changed, 94 insertions(+), 8 deletions(-)

diff --git a/src/detect-engine.c b/src/detect-engine.c
index f70d59656026..06e4b216f2c0 100644
--- a/src/detect-engine.c
+++ b/src/detect-engine.c
@@ -955,14 +955,15 @@ static char DetectBufferTypeCompareIdFunc(void *data1, uint16_t len1, void *data
     return map1->id == map2->id;
 }
 
-static void DetectBufferTypeFreeFunc(void *data)
+static void DetectBufferTypeFreeFunc(void *ctx, void *data)
 {
-    DetectBufferType *map = (DetectBufferType *)data;
-
-    if (map == NULL) {
+    if (data == NULL) {
         return;
     }
 
+    DetectBufferType *map = (DetectBufferType *)data;
+    DetectEngineCtx *de_ctx = (DetectEngineCtx *)ctx;
+
     /* Release transformation option memory, if any */
     for (int i = 0; i < map->transforms.cnt; i++) {
         if (map->transforms.transforms[i].options == NULL)
@@ -972,7 +973,8 @@ static void DetectBufferTypeFreeFunc(void *data)
                     sigmatch_table[map->transforms.transforms[i].transform].name);
             continue;
         }
-        sigmatch_table[map->transforms.transforms[i].transform].Free(NULL, map->transforms.transforms[i].options);
+        sigmatch_table[map->transforms.transforms[i].transform].Free(
+                de_ctx, map->transforms.transforms[i].options);
     }
 
     SCFree(map);
@@ -981,7 +983,7 @@ static void DetectBufferTypeFreeFunc(void *data)
 static int DetectBufferTypeInit(void)
 {
     BUG_ON(g_buffer_type_hash);
-    g_buffer_type_hash = HashListTableInit(256, DetectBufferTypeHashNameFunc,
+    g_buffer_type_hash = HashListTableInitWithCtx(256, DetectBufferTypeHashNameFunc,
             DetectBufferTypeCompareNameFunc, DetectBufferTypeFreeFunc);
     if (g_buffer_type_hash == NULL)
         return -1;
@@ -1717,7 +1719,7 @@ static void DetectBufferTypeSetupDetectEngine(DetectEngineCtx *de_ctx)
     const int size = g_buffer_type_id;
     BUG_ON(!(size > 0));
 
-    de_ctx->buffer_type_hash_name = HashListTableInit(256, DetectBufferTypeHashNameFunc,
+    de_ctx->buffer_type_hash_name = HashListTableInitWithCtx(256, DetectBufferTypeHashNameFunc,
             DetectBufferTypeCompareNameFunc, DetectBufferTypeFreeFunc);
     BUG_ON(de_ctx->buffer_type_hash_name == NULL);
     de_ctx->buffer_type_hash_id =
@@ -1759,7 +1761,7 @@ static void DetectBufferTypeFreeDetectEngine(DetectEngineCtx *de_ctx)
 {
     if (de_ctx) {
         if (de_ctx->buffer_type_hash_name)
-            HashListTableFree(de_ctx->buffer_type_hash_name);
+            HashListTableFreeWithCtx(de_ctx, de_ctx->buffer_type_hash_name);
         if (de_ctx->buffer_type_hash_id)
             HashListTableFree(de_ctx->buffer_type_hash_id);
 
diff --git a/src/util-hashlist.c b/src/util-hashlist.c
index 085a988afe76..b082f164380f 100644
--- a/src/util-hashlist.c
+++ b/src/util-hashlist.c
@@ -32,6 +32,58 @@
 #include "util-debug.h"
 #include "util-memcmp.h"
 
+HashListTable *HashListTableInitWithCtx(uint32_t size,
+        uint32_t (*Hash)(struct HashListTable_ *, void *, uint16_t),
+        char (*Compare)(void *, uint16_t, void *, uint16_t), void (*FreeWithCtx)(void *, void *))
+{
+    sc_errno = SC_OK;
+    HashListTable *ht = NULL;
+
+    if (size == 0) {
+        sc_errno = SC_EINVAL;
+        goto error;
+    }
+
+    if (Hash == NULL) {
+        sc_errno = SC_EINVAL;
+        goto error;
+    }
+
+    /* setup the filter */
+    ht = SCCalloc(1, sizeof(HashListTable));
+    if (unlikely(ht == NULL)) {
+        sc_errno = SC_ENOMEM;
+        goto error;
+    }
+    ht->array_size = size;
+    ht->Hash = Hash;
+    ht->FreeWithCtx = FreeWithCtx;
+
+    if (Compare != NULL)
+        ht->Compare = Compare;
+    else
+        ht->Compare = HashListTableDefaultCompare;
+
+    /* setup the bitarray */
+    ht->array = SCCalloc(ht->array_size, sizeof(HashListTableBucket *));
+    if (ht->array == NULL) {
+        sc_errno = SC_ENOMEM;
+        goto error;
+    }
+
+    ht->listhead = NULL;
+    ht->listtail = NULL;
+    return ht;
+
+error:
+    if (ht != NULL) {
+        if (ht->array != NULL)
+            SCFree(ht->array);
+
+        SCFree(ht);
+    }
+    return NULL;
+}
 HashListTable *HashListTableInit(uint32_t size,
         uint32_t (*Hash)(struct HashListTable_ *, void *, uint16_t),
         char (*Compare)(void *, uint16_t, void *, uint16_t), void (*Free)(void *))
@@ -85,6 +137,32 @@ HashListTable *HashListTableInit(uint32_t size,
     return NULL;
 }
 
+void HashListTableFreeWithCtx(void *ctx, HashListTable *ht)
+{
+    uint32_t i = 0;
+
+    if (ht == NULL)
+        return;
+
+    /* free the buckets */
+    for (i = 0; i < ht->array_size; i++) {
+        HashListTableBucket *hashbucket = ht->array[i];
+        while (hashbucket != NULL) {
+            HashListTableBucket *next_hashbucket = hashbucket->bucknext;
+            if (ht->FreeWithCtx != NULL)
+                ht->FreeWithCtx(ctx, hashbucket->data);
+            SCFree(hashbucket);
+            hashbucket = next_hashbucket;
+        }
+    }
+
+    /* free the array */
+    if (ht->array != NULL)
+        SCFree(ht->array);
+
+    SCFree(ht);
+}
+
 void HashListTableFree(HashListTable *ht)
 {
     uint32_t i = 0;
diff --git a/src/util-hashlist.h b/src/util-hashlist.h
index 15bd578e5319..6ae320098ae0 100644
--- a/src/util-hashlist.h
+++ b/src/util-hashlist.h
@@ -42,10 +42,16 @@ typedef struct HashListTable_ {
     uint32_t (*Hash)(struct HashListTable_ *, void *, uint16_t);
     char (*Compare)(void *, uint16_t, void *, uint16_t);
     void (*Free)(void *);
+    void (*FreeWithCtx)(void *, void *);
 } HashListTable;
 
 /* prototypes */
 HashListTable* HashListTableInit(uint32_t, uint32_t (*Hash)(struct HashListTable_ *, void *, uint16_t), char (*Compare)(void *, uint16_t, void *, uint16_t), void (*Free)(void *));
+HashListTable *HashListTableInitWithCtx(uint32_t,
+        uint32_t (*Hash)(struct HashListTable_ *, void *, uint16_t),
+        char (*Compare)(void *, uint16_t, void *, uint16_t), void (*FreeWithCtx)(void *, void *));
+
+void HashListTableFreeWithCtx(void *, HashListTable *);
 void HashListTableFree(HashListTable *);
 int HashListTableAdd(HashListTable *, void *, uint16_t);
 int HashListTableRemove(HashListTable *, void *, uint16_t);

From 2505024002eca6060de91b135fcbde92ae3f35ad Mon Sep 17 00:00:00 2001
From: Jeff Lucovsky <jlucovsky@oisf.net>
Date: Thu, 12 Sep 2024 10:51:03 -0400
Subject: [PATCH 3/6] detect/engine: Prevent double-free of  keyword hash

Issue: 2290

Defer freeing the keyword hash table until the engine context has
been freed. This eliminates a double-free from occurring.
---
 src/detect-engine.c | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/detect-engine.c b/src/detect-engine.c
index 06e4b216f2c0..0ca22a4bbeab 100644
--- a/src/detect-engine.c
+++ b/src/detect-engine.c
@@ -2671,6 +2671,7 @@ void DetectEngineCtxFree(DetectEngineCtx *de_ctx)
     DetectPortCleanupList(de_ctx, de_ctx->udp_whitelist);
 
     DetectBufferTypeFreeDetectEngine(de_ctx);
+    DetectEngineCtxFreeThreadKeywordData(de_ctx);
     SCClassConfDeinit(de_ctx);
     SCReferenceConfDeinit(de_ctx);
 

From 3503a5cc4a376f64e0683ffb57e5eda0d9165e1b Mon Sep 17 00:00:00 2001
From: Jeff Lucovsky <jlucovsky@oisf.net>
Date: Mon, 9 Sep 2024 11:06:32 -0400
Subject: [PATCH 4/6] transform: Add luaxform files

Issue: 2290

This commit adds the source files for the new transform -- luaxform.
---
 src/Makefile.am                 |   2 +
 src/detect-engine-register.c    |   2 +
 src/detect-engine-register.h    |   1 +
 src/detect-lua.c                |   4 +-
 src/detect-transform-luaxform.c | 363 ++++++++++++++++++++++++++++++++
 src/detect-transform-luaxform.h |  47 +++++
 src/util-lua-common.h           |   3 +
 7 files changed, 419 insertions(+), 3 deletions(-)
 create mode 100644 src/detect-transform-luaxform.c
 create mode 100644 src/detect-transform-luaxform.h

diff --git a/src/Makefile.am b/src/Makefile.am
index e409ce492ca4..94b20442d372 100755
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -311,6 +311,7 @@ noinst_HEADERS = \
 	detect-transform-compress-whitespace.h \
 	detect-transform-dotprefix.h \
 	detect-transform-header-lowercase.h \
+	detect-transform-luaxform.h \
 	detect-transform-md5.h \
 	detect-transform-pcrexform.h \
 	detect-transform-sha1.h \
@@ -884,6 +885,7 @@ libsuricata_c_a_SOURCES = \
 	detect-transform-compress-whitespace.c \
 	detect-transform-dotprefix.c \
 	detect-transform-header-lowercase.c \
+	detect-transform-luaxform.c \
 	detect-transform-md5.c \
 	detect-transform-pcrexform.c \
 	detect-transform-sha1.c \
diff --git a/src/detect-engine-register.c b/src/detect-engine-register.c
index 794f680dd46f..ad6cce42a292 100644
--- a/src/detect-engine-register.c
+++ b/src/detect-engine-register.c
@@ -229,6 +229,7 @@
 #include "detect-transform-casechange.h"
 #include "detect-transform-header-lowercase.h"
 #include "detect-transform-base64.h"
+#include "detect-transform-luaxform.h"
 
 #include "util-rule-vars.h"
 
@@ -685,6 +686,7 @@ void SigTableSetup(void)
     DetectTransformToUpperRegister();
     DetectTransformHeaderLowercaseRegister();
     DetectTransformFromBase64DecodeRegister();
+    DetectTransformLuaxformRegister();
 
     DetectFileHandlerRegister();
 
diff --git a/src/detect-engine-register.h b/src/detect-engine-register.h
index 7c3b5b4514b0..24b5ed94b440 100644
--- a/src/detect-engine-register.h
+++ b/src/detect-engine-register.h
@@ -316,6 +316,7 @@ enum DetectKeywordId {
     DETECT_TRANSFORM_TOUPPER,
     DETECT_TRANSFORM_HEADER_LOWERCASE,
     DETECT_TRANSFORM_FROM_BASE64,
+    DETECT_TRANSFORM_LUAXFORM,
 
     DETECT_AL_IKE_EXCH_TYPE,
     DETECT_AL_IKE_SPI_INITIATOR,
diff --git a/src/detect-lua.c b/src/detect-lua.c
index 36f045f50394..577af3154d58 100644
--- a/src/detect-lua.c
+++ b/src/detect-lua.c
@@ -57,6 +57,7 @@
 #include "util-var-name.h"
 
 #include "util-lua.h"
+#include "util-lua-common.h"
 #include "util-lua-sandbox.h"
 
 static int DetectLuaMatch (DetectEngineThreadCtx *,
@@ -126,9 +127,6 @@ void DetectLuaRegister(void)
 #define FLAG_INSTRUCTION_LIMIT_LOGGED           BIT_U32(25)
 #define FLAG_MEMORY_LIMIT_LOGGED                BIT_U32(26)
 
-#define DEFAULT_LUA_ALLOC_LIMIT       500000
-#define DEFAULT_LUA_INSTRUCTION_LIMIT 500000
-
 #if 0
 /** \brief dump stack from lua state to screen */
 void LuaDumpStack(lua_State *state)
diff --git a/src/detect-transform-luaxform.c b/src/detect-transform-luaxform.c
new file mode 100644
index 000000000000..60b9da441f96
--- /dev/null
+++ b/src/detect-transform-luaxform.c
@@ -0,0 +1,363 @@
+/* Copyright (C) 2024 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Jeff Lucovsky <jlucovsky@oisf.net>
+ *
+ * Implements the luxaform transform keyword
+ */
+
+#include "suricata-common.h"
+
+#include "detect.h"
+#include "detect-engine.h"
+#include "detect-parse.h"
+#include "detect-lua.h"
+#include "detect-transform-luaxform.h"
+#include "detect-lua-extensions.h"
+
+#include "util-lua.h"
+#include "util-lua-common.h"
+#include "util-print.h"
+
+static int DetectTransformLuaxformSetup(DetectEngineCtx *, Signature *, const char *);
+static void DetectTransformLuaxformFree(DetectEngineCtx *de_ctx, void *ptr);
+static void TransformLuaxform(InspectionBuffer *buffer, void *options);
+
+void DetectTransformLuaxformRegister(void)
+{
+    sigmatch_table[DETECT_TRANSFORM_LUAXFORM].name = "luaxform";
+    sigmatch_table[DETECT_TRANSFORM_LUAXFORM].desc =
+            "pass inspection buffer to a Lua function along with "
+            "arguments supplied to the transform";
+    sigmatch_table[DETECT_TRANSFORM_LUAXFORM].url = "/rules/transforms.html#luaxform";
+    sigmatch_table[DETECT_TRANSFORM_LUAXFORM].Transform = TransformLuaxform;
+    sigmatch_table[DETECT_TRANSFORM_LUAXFORM].Free = DetectTransformLuaxformFree;
+    sigmatch_table[DETECT_TRANSFORM_LUAXFORM].Setup = DetectTransformLuaxformSetup;
+    sigmatch_table[DETECT_TRANSFORM_LUAXFORM].flags |= SIGMATCH_QUOTES_OPTIONAL;
+}
+
+static void DetectTransformLuaxformFree(DetectEngineCtx *de_ctx, void *ptr)
+{
+    if (ptr != NULL) {
+        DetectLuaxformData *lua = (DetectLuaxformData *)ptr;
+
+        if (lua->filename)
+            SCFree(lua->filename);
+
+        if (lua->copystr)
+            SCFree(lua->copystr);
+
+        if (de_ctx) {
+            DetectUnregisterThreadCtxFuncs(de_ctx, lua, "luaxform");
+        }
+
+        SCFree(lua);
+    }
+}
+
+static int DetectTransformLuaxformSetupPrime(
+        DetectEngineCtx *de_ctx, DetectLuaxformData *ld, const Signature *s)
+{
+    lua_State *luastate = SCLuaSbStateNew(ld->alloc_limit, ld->instruction_limit);
+    if (luastate == NULL)
+        return -1;
+    if (ld->allow_restricted_functions) {
+        luaL_openlibs(luastate);
+    } else {
+        SCLuaSbLoadLibs(luastate);
+    }
+
+    int status = luaL_loadfile(luastate, ld->filename);
+    if (status) {
+        SCLogError("couldn't load file: %s", lua_tostring(luastate, -1));
+        goto error;
+    }
+
+    /* prime the script (or something) */
+    if (lua_pcall(luastate, 0, 0, 0) != 0) {
+        SCLogError("couldn't prime file: %s", lua_tostring(luastate, -1));
+        goto error;
+    }
+
+    lua_getglobal(luastate, "transform");
+    if (lua_type(luastate, -1) != LUA_TFUNCTION) {
+        SCLogError("no transform function in script");
+        goto error;
+    }
+    lua_pop(luastate, 1);
+
+    /* init -- optional entry point */
+    lua_getglobal(luastate, "init");
+    if (lua_type(luastate, -1) != LUA_TFUNCTION) {
+        goto early_return;
+    }
+
+    lua_newtable(luastate); /* stack at -1 */
+    if (lua_gettop(luastate) == 0 || lua_type(luastate, 2) != LUA_TTABLE) {
+        SCLogError("no table setup");
+        goto error;
+    }
+
+    lua_pushliteral(luastate, "script_api_ver"); /* stack at -2 */
+    lua_pushnumber(luastate, 1);                 /* stack at -3 */
+    lua_settable(luastate, -3);
+
+    if (lua_pcall(luastate, 1, 1, 0) != 0) {
+        SCLogError("couldn't run script 'init' function: %s", lua_tostring(luastate, -1));
+        goto error;
+    }
+
+    /* process returns from script */
+    if (lua_gettop(luastate) == 0) {
+        SCLogError("init function in script should return table, nothing returned");
+        goto error;
+    }
+    if (lua_type(luastate, 1) != LUA_TTABLE) {
+        SCLogError("init function in script should return table, returned is not table");
+        goto error;
+    }
+
+    LuaRegisterExtensions(luastate);
+    /* pop the table */
+    lua_pop(luastate, 1);
+
+early_return:
+    SCLuaSbStateClose(luastate);
+    return 0;
+
+error:
+    SCLuaSbStateClose(luastate);
+    return -1;
+}
+
+static DetectLuaxformData *DetectLuaxformParse(DetectEngineCtx *de_ctx, const char *str)
+{
+    DetectLuaxformData *lua = NULL;
+
+    /* We have a correct lua option */
+    lua = SCCalloc(1, sizeof(DetectLuaxformData));
+    if (unlikely(lua == NULL)) {
+        FatalError("unable to allocate memory for Lua transform: %s", str);
+    }
+
+    lua->copystr = strdup(str);
+
+    int count = 0;
+    char *token = strtok(lua->copystr, ",");
+    while (token != NULL && count < LUAXFORM_MAX_ARGS) {
+        lua->args[count++] = token;
+        token = strtok(NULL, ",");
+    }
+
+    if (count == 0) {
+        SCLogError("Lua script name not supplied");
+        goto error;
+    }
+
+    lua->arg_count = count - 1;
+
+    /* get full filename */
+    lua->filename = DetectLoadCompleteSigPath(de_ctx, lua->args[0]);
+    if (lua->filename == NULL) {
+        goto error;
+    }
+
+    return lua;
+
+error:
+    if (lua != NULL)
+        DetectTransformLuaxformFree(de_ctx, lua);
+    return NULL;
+}
+
+static void *DetectLuaxformThreadInit(void *data)
+{
+    DetectLuaxformData *lua = (DetectLuaxformData *)data;
+    BUG_ON(lua == NULL);
+
+    DetectLuaThreadData *t = SCCalloc(1, sizeof(DetectLuaThreadData));
+    if (unlikely(t == NULL)) {
+        FatalError("unable to allocate luaxform context memory");
+    }
+
+    t->luastate = SCLuaSbStateNew(lua->alloc_limit, lua->instruction_limit);
+    if (t->luastate == NULL) {
+        SCLogError("luastate pool depleted");
+        goto error;
+    }
+
+    if (lua->allow_restricted_functions) {
+        luaL_openlibs(t->luastate);
+    } else {
+        SCLuaSbLoadLibs(t->luastate);
+    }
+
+    LuaRegisterExtensions(t->luastate);
+
+    int status = luaL_loadfile(t->luastate, lua->filename);
+    if (status) {
+        SCLogError("couldn't load file: %s", lua_tostring(t->luastate, -1));
+        goto error;
+    }
+
+    /* prime the script (or something) */
+    if (lua_pcall(t->luastate, 0, 0, 0) != 0) {
+        SCLogError("couldn't prime file: %s", lua_tostring(t->luastate, -1));
+        goto error;
+    }
+
+    return (void *)t;
+
+error:
+    if (t->luastate != NULL)
+        SCLuaSbStateClose(t->luastate);
+    SCFree(t);
+    return NULL;
+}
+
+static void DetectLuaxformThreadFree(void *ctx)
+{
+    if (ctx != NULL) {
+        DetectLuaxformThreadData *t = (DetectLuaxformThreadData *)ctx;
+        if (t->luastate != NULL)
+            SCLuaSbStateClose(t->luastate);
+        SCFree(t);
+    }
+}
+
+/**
+ *  \internal
+ *  \brief Apply the luaxform keyword to the last pattern match
+ *  \param de_ctx detection engine ctx
+ *  \param s signature
+ *  \param str lua filename and optional args
+ *  \retval 0 ok
+ *  \retval -1 failure
+ */
+static int DetectTransformLuaxformSetup(DetectEngineCtx *de_ctx, Signature *s, const char *str)
+{
+    SCEnter();
+
+    /* First check if Lua rules are enabled, by default Lua in rules
+     * is disabled. */
+    int enabled = 0;
+    (void)ConfGetBool("security.lua.allow-rules", &enabled);
+    if (!enabled) {
+        SCLogError("Lua rules disabled by security configuration: security.lua.allow-rules");
+        SCReturnInt(-1);
+    }
+
+    DetectLuaxformData *lua = DetectLuaxformParse(de_ctx, str);
+    if (lua == NULL)
+        goto error;
+
+    /* Load lua sandbox configurations */
+    intmax_t lua_alloc_limit = DEFAULT_LUA_ALLOC_LIMIT;
+    intmax_t lua_instruction_limit = DEFAULT_LUA_INSTRUCTION_LIMIT;
+    int allow_restricted_functions = 0;
+    (void)ConfGetInt("security.lua.max-bytes", &lua_alloc_limit);
+    (void)ConfGetInt("security.lua.max-instructions", &lua_instruction_limit);
+    (void)ConfGetBool("security.lua.allow-restricted-functions", &allow_restricted_functions);
+
+    lua->alloc_limit = lua_alloc_limit;
+    lua->instruction_limit = lua_instruction_limit;
+    lua->allow_restricted_functions = allow_restricted_functions;
+
+    if (DetectTransformLuaxformSetupPrime(de_ctx, lua, s) == -1) {
+        goto error;
+    }
+
+    lua->thread_ctx_id = DetectRegisterThreadCtxFuncs(
+            de_ctx, "luaxform", DetectLuaxformThreadInit, (void *)lua, DetectLuaxformThreadFree, 0);
+    if (lua->thread_ctx_id == -1)
+        goto error;
+
+    if (0 == DetectSignatureAddTransform(s, DETECT_TRANSFORM_LUAXFORM, lua))
+        SCReturnInt(0);
+
+error:
+
+    if (lua != NULL)
+        DetectTransformLuaxformFree(de_ctx, lua);
+    SCReturnInt(-1);
+}
+
+static void TransformLuaxform(InspectionBuffer *buffer, void *options)
+{
+    if (buffer->inspect_len == 0) {
+        return;
+    }
+
+    DetectLuaxformData *lua = options;
+    DetectLuaThreadData *tlua = (DetectLuaThreadData *)DetectThreadCtxGetKeywordThreadCtx(
+            buffer->det_ctx, lua->thread_ctx_id);
+    if (tlua == NULL) {
+        return;
+    }
+
+    lua_getglobal(tlua->luastate, "transform");
+
+    const uint8_t *input = buffer->inspect;
+    const uint32_t input_len = buffer->inspect_len;
+    /* Lua script args are: buffer length, buffer, arg count, args */
+    LuaPushInteger(tlua->luastate, (lua_Integer)input_len);
+    LuaPushStringBuffer(tlua->luastate, input, (size_t)input_len);
+    LuaPushInteger(tlua->luastate, (lua_Integer)lua->arg_count);
+
+    /*
+     * Add provided arguments for lua script (these are optionally
+     * provided by the rule writer).
+     *
+     * Start at offset 1 (arg[0] is the lua script filename)
+     */
+    lua_newtable(tlua->luastate);
+    for (int i = 1; i < lua->arg_count + 1; i++) {
+        LuaPushInteger(tlua->luastate, i);
+        lua_pushstring(tlua->luastate, lua->args[i]);
+        lua_settable(tlua->luastate, -3);
+    }
+
+    SCLuaSbResetInstructionCounter(tlua->luastate);
+
+    if (LUA_OK != lua_pcall(tlua->luastate, 4, 2, 0)) {
+        SCLogDebug("error calling lua script: %s", lua_tostring(tlua->luastate, -1));
+    } else {
+        /* Lua transform functions must return 2 values: buffer and length */
+        int return_value_count = lua_gettop(tlua->luastate);
+        if (return_value_count != 2) {
+            SCLogDebug("Error: expected 2 return values but got %d", return_value_count);
+            lua_pop(tlua->luastate, return_value_count);
+            return;
+        }
+        if (lua_isstring(tlua->luastate, -2)) {
+            const char *transformed_buffer = lua_tostring(tlua->luastate, -2);
+            int transformed_buffer_byte_count = lua_tointeger(tlua->luastate, -1);
+            if (transformed_buffer != NULL && transformed_buffer_byte_count > 0)
+                InspectionBufferCopy(
+                        buffer, (uint8_t *)transformed_buffer, transformed_buffer_byte_count);
+            SCLogDebug("transform returns [nbytes %d] \"%p\"", transformed_buffer_byte_count,
+                    transformed_buffer);
+            // PrintRawDataFp(stdout, (const uint8_t *)transformed_buffer,
+            // transformed_buffer_byte_count);
+        }
+    }
+
+    lua_pop(tlua->luastate, 2); // Pop the result string/length
+}
diff --git a/src/detect-transform-luaxform.h b/src/detect-transform-luaxform.h
new file mode 100644
index 000000000000..9a5cb8700f63
--- /dev/null
+++ b/src/detect-transform-luaxform.h
@@ -0,0 +1,47 @@
+/* Copyright (C) 2024 Open Information Security Foundation
+ *
+ * You can copy, redistribute or modify this Program under the terms of
+ * the GNU General Public License version 2 as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * version 2 along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
+ * 02110-1301, USA.
+ */
+
+/**
+ * \file
+ *
+ * \author Jeff Lucovsky <jlucovsky@oisf.net>
+ */
+
+#ifndef SURICATA_DETECT_TRANSFORM_LUAXFORM_H
+#define SURICATA_DETECT_TRANSFORM_LUAXFORM_H
+
+/* prototypes */
+void DetectTransformLuaxformRegister(void);
+
+#define LUAXFORM_MAX_ARGS 10
+
+typedef struct DetectLuaxformData {
+    int thread_ctx_id;
+    int allow_restricted_functions;
+    int arg_count;
+    uint64_t alloc_limit;
+    uint64_t instruction_limit;
+    char *filename;
+    char *copystr;
+    char *args[LUAXFORM_MAX_ARGS];
+} DetectLuaxformData;
+
+typedef struct DetectLuaxformThreadData {
+    lua_State *luastate;
+} DetectLuaxformThreadData;
+
+#endif /* SURICATA_DETECT_TRANSFORM_LUAXFORM_H */
diff --git a/src/util-lua-common.h b/src/util-lua-common.h
index 5d6ea41f4be4..4834f4fa7af9 100644
--- a/src/util-lua-common.h
+++ b/src/util-lua-common.h
@@ -24,6 +24,9 @@
 #ifndef SURICATA_UTIL_LUA_COMMON_H
 #define SURICATA_UTIL_LUA_COMMON_H
 
+#define DEFAULT_LUA_ALLOC_LIMIT       500000
+#define DEFAULT_LUA_INSTRUCTION_LIMIT 500000
+
 int LuaCallbackError(lua_State *luastate, const char *msg);
 const char *LuaGetStringArgument(lua_State *luastate, int argc);
 

From f42d6ff4f552fb367e86aa4e5aa814984eec2f62 Mon Sep 17 00:00:00 2001
From: Jeff Lucovsky <jlucovsky@oisf.net>
Date: Wed, 18 Sep 2024 08:18:07 -0400
Subject: [PATCH 5/6] doc: Document luaxform transform

Issue: 2290
---
 doc/userguide/configuration/suricata-yaml.rst |   4 +-
 doc/userguide/lua/lua-functions.rst           |  15 +--
 doc/userguide/lua/lua-usage.rst               |  17 ++-
 doc/userguide/rules/lua-detection.rst         |  19 +++-
 doc/userguide/rules/transforms.rst            | 104 ++++++++++++++++++
 doc/userguide/upgrade.rst                     |   1 +
 6 files changed, 143 insertions(+), 17 deletions(-)

diff --git a/doc/userguide/configuration/suricata-yaml.rst b/doc/userguide/configuration/suricata-yaml.rst
index 3dfc950ec641..a6310c430ddc 100644
--- a/doc/userguide/configuration/suricata-yaml.rst
+++ b/doc/userguide/configuration/suricata-yaml.rst
@@ -2829,8 +2829,8 @@ Lua
 ~~~
 
 Suricata 8.0 sandboxes Lua rules by default. The restrictions on the sandbox for Lua rules can be
-modified in the ``security.lua`` section of the configuration file.  Additionally, Lua rules 
-can be completely disabled the same as the Suricata 7.0 default:
+modified in the ``security.lua`` section of the configuration file.  This section also applies to
+Lua transforms Additionally, Lua rules can be completely disabled the same as the Suricata 7.0 default:
 
 ::
 
diff --git a/doc/userguide/lua/lua-functions.rst b/doc/userguide/lua/lua-functions.rst
index 92473d52c35e..6d598e0d7a6a 100644
--- a/doc/userguide/lua/lua-functions.rst
+++ b/doc/userguide/lua/lua-functions.rst
@@ -6,7 +6,8 @@ Lua functions
 Differences between `output` and `detect`:
 ------------------------------------------
 
-Currently, the ``needs`` key initialization varies, depending on what is the goal of the script: output or detection.
+Currently, the ``needs`` key initialization varies, depending on what is the goal of the script: output or detection. The
+Lua script for the ``luaxform`` transform **does not use ``needs``**.
 
 If the script is for detection, the ``needs`` initialization should be as seen in the example below (see :ref:`lua-detection` for a complete example of a detection script):
 
@@ -812,7 +813,7 @@ Example:
           return 0
       end
   end
-  
+
 HasshServerGet
 ~~~~~~~~~~~~~~
 
@@ -828,7 +829,7 @@ Example:
           return 0
       end
   end
-  
+
 HasshServerGetString
 ~~~~~~~~~~~~~~~~~~~~
 
@@ -998,7 +999,7 @@ index so in our case we need to use 0.
          SCFlowintSet(0, a + 1)
      else
          SCFlowintSet(0, 1)
-     end 
+     end
 
 SCFlowintGet
 ~~~~~~~~~~~~
@@ -1031,7 +1032,7 @@ SCFlowvarSet
 Set a Flowvar. First parameter is the index, second is the data
 and third is the length of data.
 
-You can use it to set string 
+You can use it to set string
 
 ::
 
@@ -1041,7 +1042,7 @@ You can use it to set string
      needs["flowvar"] = {"cnt"}
      return needs
  end
- 
+
  function match(args)
      a = SCFlowvarGet(0);
      if a then
@@ -1050,7 +1051,7 @@ You can use it to set string
      else
          a = tostring(1)
          SCFlowvarSet(0, a, #a)
-     end 
+     end
 
 Misc
 ----
diff --git a/doc/userguide/lua/lua-usage.rst b/doc/userguide/lua/lua-usage.rst
index 19946db5e54f..476453814d14 100644
--- a/doc/userguide/lua/lua-usage.rst
+++ b/doc/userguide/lua/lua-usage.rst
@@ -1,20 +1,27 @@
 Lua usage in Suricata
 =====================
 
-Lua scripting can be used in two components of Suricata. The first is in
-output and the second one in rules in the detection engine.
+Lua scripting can be used in two components of Suricata:
+
+  * Output
+  * Detection: ``lua`` keyword and ``luaxform`` transform
 
 Both features are using a list of functions to access the data extracted by
 Suricata. You can get the list of functions in the :ref:`lua-functions` page.
 
-.. note:: Currently, there is a difference in the ``needs`` key in the ``init`` function, depending on what is the usage: ``output`` or ``detection``. The list of available functions may also differ.
+.. note:: Currently, there is a difference in the ``needs`` key in the ``init`` function, depending on what is the usage: ``output`` or ``detection``. The list of available functions may also differ. The ``luaxform`` doesn't use the ``needs`` key.
 
 Lua output
 ----------
 
-Lua can be used to write arbitrary output. See :ref:`lua-output` for more information.
+Lua scripts can be used to write arbitrary output. See :ref:`lua-output` for more information.
 
 Lua detection
 -------------
 
-Lua script can be used as a filter condition in signatures. See :ref:`lua-detection` for more information.
+Lua scripts can be used as a filter condition in signatures. See :ref:`lua-detection` for more information.
+
+Lua transform
+-------------
+
+The ``luaxform`` transform can be used in signatures.  See :ref:`lua-transform` for more information.
diff --git a/doc/userguide/rules/lua-detection.rst b/doc/userguide/rules/lua-detection.rst
index 0f2011987ec0..cb2535a41d87 100644
--- a/doc/userguide/rules/lua-detection.rst
+++ b/doc/userguide/rules/lua-detection.rst
@@ -3,10 +3,18 @@
 Lua Scripting for Detection
 ===========================
 
+There are 2 ways that Lua can be used with detection. These are
+
+* ``lua`` rule keyword.
+* ``luaxform`` transform.
+
 .. note:: Lua is disabled by default for use in rules, it must be
           enabled in the configuration file. See the ``security.lua``
           section of ``suricata.yaml`` and enable ``allow-rules``.
 
+Lua Rule Keyword
+^^^^^^^^^^^^^^^^
+
 Syntax:
 
 ::
@@ -103,8 +111,13 @@ Entire script:
 
   return 0
 
-Sandbox and Available functions
--------------------------------
+Lua Transform: ``luaxform``
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+More details in :ref:`lua-transform`
+
+Lua Sandbox and Available functions
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 Lua rule scripts are run in a sandbox environment the applies the
 following restrictions:
@@ -140,7 +153,7 @@ Of note, the following standard libraries are not available:
 
 This behavior can be modified via the ``security.lua`` section of :ref:`suricata-yaml-lua-config`
 
-.. note:: Suricata 8.0 has moved to Lua 5.4 and has builtin support for bitwise and utf8 operations now.
+.. note:: Suricata 8.0 has moved to Lua 5.4 and now has builtin support for bitwise and utf8 operations.
 
 A comprehensive list of existing lua functions - with examples - can
 be found at :ref:`lua-functions` (some of them, however, work only for
diff --git a/doc/userguide/rules/transforms.rst b/doc/userguide/rules/transforms.rst
index e536757f29f7..bfd21bbb8ba8 100644
--- a/doc/userguide/rules/transforms.rst
+++ b/doc/userguide/rules/transforms.rst
@@ -243,3 +243,107 @@ This example transforms `"Zm 9v Ym Fy"` to `"foobar"`::
 
        content:"/?arg=Zm 9v Ym Fy"; from_base64: offset 6, mode rfc2045; \
        content:"foobar";
+
+.. _lua-transform:
+
+luaxform
+--------
+
+This transform allows a Lua script to apply a transformation
+to a buffer.
+
+Lua scripts that are used for transformations *must* contain a function
+named ``transform``.
+
+Lua transforms can be passed optional arguments -- see the examples below -- but they
+are not required to do so. Optional arguments are included with the transform (see example
+below).
+
+Note that the arguments and values are passed without validation
+nor interpretation. There is a maximum of 10 arguments.
+
+The Lua ``transform`` function receive parameters:
+
+  * `input-length` The number of bytes in the buffer provided to the transform
+  * `input` The buffer provided to the transform
+  * `argument` The number of arguments provided in the following parameters. If there are
+    no arguments to the Lua transform, this value will be `0`.
+  * `arguments` The list of arguments.
+
+The Lua ``transform`` must return two values
+
+  * `transformed-buffer` The buffer with the transformed bytes.
+  * `transformed-buffer-byte-count` The number of bytes in the transformed buffer.
+
+This example supplies the HTTP data to a Lua transform with the transform
+results being checked with `content`.
+
+Example::
+
+    alert http any any -> any any (msg:"Lua Xform example"; flow:established;  \
+    file.data; luaxform:./lua/lua-transform.lua; content: "abc"; sid: 2;)
+
+
+This example supplies the HTTP data to a Lua transform with the transform
+with arguments that specify the offset and byte count for the transform.
+The resulting buffer is then checked with a `content` match.
+
+Example::
+
+    alert http any any -> any any (msg:"Lua Xform example"; flow:established; \
+    file.data; luaxform:./lua/lua-transform.lua,  bytes 12, offset 13; content: "abc"; sid: 1;)
+
+The following Lua script shows a transform that handles arguments: `bytes` and `offset` and uses
+those values (or defaults, if there are no arguments) for applying the uppercase transform to
+the buffer.
+
+.. code-block:: lua
+
+   function init (args)
+        local needs = {}
+        return needs
+   end
+
+   local function get_value(item, key)
+       if string.find(item, key) then
+           local _, value = string.match(item, "(%a+)%s*(%d*)")
+           if value ~= "" then
+               return tonumber(value)
+           end
+       end
+
+       return nil
+   end
+
+   -- Arguments supported
+   local bytes_key = "bytes"
+   local offset_key = "offset"
+   function transform(input_len, input, argc, args)
+       local bytes = input_len
+       local offset = 0
+
+       -- Look for optional bytes and offset arguments
+       for i, item in ipairs(args) do
+           local value = get_value(item, bytes_key)
+           if value ~= nil then
+               bytes = value
+           else
+               local value = get_value(item, offset_key)
+               if value ~= nil then
+                   offset = value
+               end
+           end
+       end
+       local str_len = #input
+       if offset < 0 or offset > str_len then
+           print("offset is out of bounds: " .. offset)
+           return nil
+       end
+       str_len = str_len - offset
+       if bytes < 0 or bytes > str_len then
+           print("invalid bytes " ..  bytes .. " or bytes > length " .. bytes .. " length " .. str_len)
+           return nil
+       end
+       local sub = string.sub(input, offset + 1, offset + bytes)
+       return string.upper(sub), bytes
+   end
diff --git a/doc/userguide/upgrade.rst b/doc/userguide/upgrade.rst
index 389cca88cf40..766287652ab5 100644
--- a/doc/userguide/upgrade.rst
+++ b/doc/userguide/upgrade.rst
@@ -77,6 +77,7 @@ Major changes
     - sip.to
     - sip.content_type
     - sip.content_length
+- New transform ``luaxform`` that uses a Lua script for sticky buffer transformation.  More details in :ref:`lua-transform`
 
 Removals
 ~~~~~~~~

From 7b8151b693befaebfd01c399479312d7d109709f Mon Sep 17 00:00:00 2001
From: Jeff Lucovsky <jlucovsky@oisf.net>
Date: Fri, 20 Sep 2024 09:57:53 -0400
Subject: [PATCH 6/6] misc: Eliminate compiler warnings

Issue: 2290

Fixup the macro usage to eliminate compiler warnings.
---
 src/util-magic.c  | 2 ++
 src/util-memcmp.c | 3 +--
 src/util-spm.c    | 6 +++---
 3 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/util-magic.c b/src/util-magic.c
index 63c991cc92d8..17b540015911 100644
--- a/src/util-magic.c
+++ b/src/util-magic.c
@@ -120,11 +120,13 @@ char *MagicThreadLookup(magic_t *ctx, const uint8_t *buf, uint32_t buflen)
 
 #ifdef UNITTESTS
 
+#if 0
 #if defined OS_FREEBSD || defined OS_DARWIN
 #define MICROSOFT_OFFICE_DOC "OLE 2 Compound Document"
 #else
 #define MICROSOFT_OFFICE_DOC "Microsoft Office Document"
 #endif
+#endif
 
 /** \test magic lib calls -- init */
 static int MagicInitTest01(void)
diff --git a/src/util-memcmp.c b/src/util-memcmp.c
index 7113b82dd60c..40f4fd67d63a 100644
--- a/src/util-memcmp.c
+++ b/src/util-memcmp.c
@@ -154,11 +154,10 @@ static int MemcmpTest13 (void)
 
 #include "util-cpu.h"
 
-#define TEST_RUNS 1000000
-
 static int MemcmpTest14 (void)
 {
 #ifdef PROFILING
+#define TEST_RUNS 1000000
     uint64_t ticks_start = 0;
     uint64_t ticks_end = 0;
     const char *a[] = { "0123456789012345", "abc", "abcdefghij", "suricata", "test", "xyz", "rrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrrr", "abcdefghijklmnopqrstuvwxyz", NULL };
diff --git a/src/util-spm.c b/src/util-spm.c
index 63510596c7a3..dce8c737e1f9 100644
--- a/src/util-spm.c
+++ b/src/util-spm.c
@@ -267,9 +267,6 @@ uint8_t *BoyerMooreNocaseSearch(const uint8_t *text, uint32_t textlen,
  *  #define ENABLE_SEARCH_STATS 1
  */
 
-/* Number of times to repeat the search (for stats) */
-#define STATS_TIMES 1000000
-
 /**
  * \brief Unittest helper function wrappers for the search algorithms
  * \param text pointer to the buffer to search in
@@ -403,6 +400,9 @@ static uint8_t *BoyerMooreNocaseWrapper(uint8_t *text, uint8_t *in_needle, int t
 }
 
 #ifdef ENABLE_SEARCH_STATS
+/* Number of times to repeat the search (for stats) */
+#define STATS_TIMES 1000000
+
 /**
  * \brief Unittest helper function wrappers for the search algorithms
  * \param text pointer to the buffer to search in