From b3490b46fdbe20cd3be0efadf6175abdee400b6c Mon Sep 17 00:00:00 2001 From: Juan Pablo Tosso Date: Sun, 6 Aug 2023 07:10:57 +0200 Subject: [PATCH] implement https mime (#850) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * implement https mime * refactor formatter interfaces * fix tinygo * Update experimental/plugins/auditlog_formatter_test.go Co-authored-by: José Carlos Chávez * minor lint * Update internal/auditlog/formats.go Co-authored-by: José Carlos Chávez * minor lint * Update experimental/plugins/plugintypes/auditlog.go Co-authored-by: José Carlos Chávez * Update internal/auditlog/formats_json.go Co-authored-by: José Carlos Chávez * Update internal/auditlog/formats_json.go Co-authored-by: José Carlos Chávez * add more tests * fix test * fix https formatter * remove factory for formatter --------- Co-authored-by: José Carlos Chávez --- experimental/plugins/auditlog.go | 4 +- .../plugins/auditlog_formatter_test.go | 15 ++++- experimental/plugins/plugintypes/auditlog.go | 8 ++- internal/auditlog/concurrent_writer.go | 2 +- internal/auditlog/concurrent_writer_test.go | 4 +- internal/auditlog/formats.go | 10 ++- internal/auditlog/formats_json.go | 22 +++++-- internal/auditlog/formats_json_test.go | 7 +- internal/auditlog/formats_test.go | 7 +- internal/auditlog/https_writer.go | 4 +- internal/auditlog/https_writer_test.go | 64 ++++++++++++++----- internal/auditlog/init.go | 6 +- internal/auditlog/init_tinygo.go | 6 +- internal/auditlog/logger.go | 4 +- internal/auditlog/logger_test.go | 11 +++- internal/auditlog/noop_formater.go | 7 +- internal/auditlog/serial_writer.go | 2 +- internal/auditlog/serial_writer_test.go | 4 +- 18 files changed, 133 insertions(+), 54 deletions(-) diff --git a/experimental/plugins/auditlog.go b/experimental/plugins/auditlog.go index 420033529..4e33ec300 100644 --- a/experimental/plugins/auditlog.go +++ b/experimental/plugins/auditlog.go @@ -14,6 +14,6 @@ func RegisterAuditLogWriter(name string, writerFactory func() plugintypes.AuditL } // RegisterAuditLogFormatter registers a new audit log formatter. -func RegisterAuditLogFormatter(name string, f func(plugintypes.AuditLog) ([]byte, error)) { - auditlog.RegisterFormatter(name, f) +func RegisterAuditLogFormatter(name string, format plugintypes.AuditLogFormatter) { + auditlog.RegisterFormatter(name, format) } diff --git a/experimental/plugins/auditlog_formatter_test.go b/experimental/plugins/auditlog_formatter_test.go index f97721156..6b0186b39 100644 --- a/experimental/plugins/auditlog_formatter_test.go +++ b/experimental/plugins/auditlog_formatter_test.go @@ -14,12 +14,21 @@ import ( "github.com/corazawaf/coraza/v3/experimental/plugins/plugintypes" ) +type testFormatter struct{} + +func (testFormatter) Format(al plugintypes.AuditLog) ([]byte, error) { + return []byte(al.Transaction().ID()), nil +} + +func (testFormatter) MIME() string { + return "sample" +} + // ExampleRegisterAuditLogFormatter shows how to register a custom audit log formatter // and tests the output of the formatter. func ExampleRegisterAuditLogFormatter() { - plugins.RegisterAuditLogFormatter("txid", func(al plugintypes.AuditLog) ([]byte, error) { - return []byte(al.Transaction().ID()), nil - }) + + plugins.RegisterAuditLogFormatter("txid", &testFormatter{}) w, err := coraza.NewWAF( coraza.NewWAFConfig(). diff --git a/experimental/plugins/plugintypes/auditlog.go b/experimental/plugins/plugintypes/auditlog.go index 2a64043ac..b64911da1 100644 --- a/experimental/plugins/plugintypes/auditlog.go +++ b/experimental/plugins/plugintypes/auditlog.go @@ -128,5 +128,9 @@ type AuditLogWriter interface { Close() error } -// AuditLogFormatter formats an audit log to a byte slice. -type AuditLogFormatter func(AuditLog) ([]byte, error) +// AuditLogFormatter serializes an AuditLog into a byte slice. +// It is used to construct the formatted audit log. +type AuditLogFormatter interface { + Format(AuditLog) ([]byte, error) + MIME() string +} diff --git a/internal/auditlog/concurrent_writer.go b/internal/auditlog/concurrent_writer.go index bb69fd851..fab2382cc 100644 --- a/internal/auditlog/concurrent_writer.go +++ b/internal/auditlog/concurrent_writer.go @@ -67,7 +67,7 @@ func (cl concurrentWriter) Write(al plugintypes.AuditLog) error { return err } - formattedAL, err := cl.formatter(al) + formattedAL, err := cl.formatter.Format(al) if err != nil { return err } diff --git a/internal/auditlog/concurrent_writer_test.go b/internal/auditlog/concurrent_writer_test.go index 94c16dff2..1cbde6613 100644 --- a/internal/auditlog/concurrent_writer_test.go +++ b/internal/auditlog/concurrent_writer_test.go @@ -38,7 +38,7 @@ func TestConcurrentWriterFailsOnInit(t *testing.T) { config.Dir = t.TempDir() config.FileMode = fs.FileMode(0777) config.DirMode = fs.FileMode(0777) - config.Formatter = jsonFormatter + config.Formatter = &jsonFormatter{} writer := &concurrentWriter{} if err := writer.Init(config); err == nil { @@ -57,7 +57,7 @@ func TestConcurrentWriterWrites(t *testing.T) { Dir: dir, FileMode: fs.FileMode(0777), DirMode: fs.FileMode(0777), - Formatter: jsonFormatter, + Formatter: &jsonFormatter{}, } ts := time.Now() expectedLog := &Log{ diff --git a/internal/auditlog/formats.go b/internal/auditlog/formats.go index bb8922764..82ad71d05 100644 --- a/internal/auditlog/formats.go +++ b/internal/auditlog/formats.go @@ -28,7 +28,9 @@ import ( "github.com/corazawaf/coraza/v3/types" ) -func nativeFormatter(al plugintypes.AuditLog) ([]byte, error) { +type nativeFormatter struct{} + +func (nativeFormatter) Format(al plugintypes.AuditLog) ([]byte, error) { boundaryPrefix := fmt.Sprintf("--%s-", utils.RandomString(10)) var res strings.Builder @@ -102,6 +104,10 @@ func nativeFormatter(al plugintypes.AuditLog) ([]byte, error) { return []byte(res.String()), nil } +func (nativeFormatter) MIME() string { + return "application/x-coraza-auditlog-native" +} + var ( - _ plugintypes.AuditLogFormatter = nativeFormatter + _ plugintypes.AuditLogFormatter = (*nativeFormatter)(nil) ) diff --git a/internal/auditlog/formats_json.go b/internal/auditlog/formats_json.go index 5137663b2..7d96c26ff 100644 --- a/internal/auditlog/formats_json.go +++ b/internal/auditlog/formats_json.go @@ -15,8 +15,9 @@ import ( "github.com/corazawaf/coraza/v3/experimental/plugins/plugintypes" ) -// Coraza format -func jsonFormatter(al plugintypes.AuditLog) ([]byte, error) { +type jsonFormatter struct{} + +func (jsonFormatter) Format(al plugintypes.AuditLog) ([]byte, error) { jsdata, err := json.Marshal(al) if err != nil { return nil, err @@ -24,8 +25,13 @@ func jsonFormatter(al plugintypes.AuditLog) ([]byte, error) { return jsdata, nil } -// Coraza legacy json format -func legacyJSONFormatter(al plugintypes.AuditLog) ([]byte, error) { +func (jsonFormatter) MIME() string { + return "application/json; charset=utf-8" +} + +type legacyJSONFormatter struct{} + +func (_ legacyJSONFormatter) Format(al plugintypes.AuditLog) ([]byte, error) { al2 := logLegacy{ Transaction: logLegacyTransaction{ Time: al.Transaction().Timestamp(), @@ -92,7 +98,11 @@ func legacyJSONFormatter(al plugintypes.AuditLog) ([]byte, error) { return jsdata, nil } +func (_ legacyJSONFormatter) MIME() string { + return "application/json; charset=utf-8" +} + var ( - _ plugintypes.AuditLogFormatter = jsonFormatter - _ plugintypes.AuditLogFormatter = legacyJSONFormatter + _ plugintypes.AuditLogFormatter = (*jsonFormatter)(nil) + _ plugintypes.AuditLogFormatter = (*legacyJSONFormatter)(nil) ) diff --git a/internal/auditlog/formats_json_test.go b/internal/auditlog/formats_json_test.go index d22065789..7e0ca684f 100644 --- a/internal/auditlog/formats_json_test.go +++ b/internal/auditlog/formats_json_test.go @@ -8,6 +8,7 @@ package auditlog import ( "encoding/json" + "strings" "testing" ) @@ -48,10 +49,14 @@ func TestModsecBoundary(t *testing.T) { func TestLegacyFormatter(t *testing.T) { al := createAuditLog() - data, err := legacyJSONFormatter(al) + f := &legacyJSONFormatter{} + data, err := f.Format(al) if err != nil { t.Error(err) } + if !strings.Contains(f.MIME(), "json") { + t.Errorf("failed to match MIME, expected json and got %s", f.MIME()) + } var legacyAl logLegacy if err := json.Unmarshal(data, &legacyAl); err != nil { t.Error(err) diff --git a/internal/auditlog/formats_test.go b/internal/auditlog/formats_test.go index ac8d0eb7a..68712a2d1 100644 --- a/internal/auditlog/formats_test.go +++ b/internal/auditlog/formats_test.go @@ -5,6 +5,7 @@ package auditlog import ( "bytes" + "strings" "testing" "github.com/corazawaf/coraza/v3/experimental/plugins/plugintypes" @@ -13,10 +14,14 @@ import ( func TestNativeFormatter(t *testing.T) { al := createAuditLog() - data, err := nativeFormatter(al) + f := &nativeFormatter{} + data, err := f.Format(al) if err != nil { t.Error(err) } + if !strings.Contains(f.MIME(), "x-coraza-auditlog-native") { + t.Errorf("failed to match MIME, expected json and got %s", f.MIME()) + } // Log contains random strings, do a simple sanity check if !bytes.Contains(data, []byte("[02/Jan/2006:15:04:20 -0700] 123 0 0")) { t.Errorf("failed to match log, \ngot: %s\n", string(data)) diff --git a/internal/auditlog/https_writer.go b/internal/auditlog/https_writer.go index b3f9f4b8a..1ff452bb2 100644 --- a/internal/auditlog/https_writer.go +++ b/internal/auditlog/https_writer.go @@ -41,7 +41,7 @@ func (h *httpsWriter) Init(c plugintypes.AuditLogConfig) error { } func (h *httpsWriter) Write(al plugintypes.AuditLog) error { - body, err := h.formatter(al) + body, err := h.formatter.Format(al) if err != nil { return err } @@ -51,7 +51,7 @@ func (h *httpsWriter) Write(al plugintypes.AuditLog) error { return err } req.Header.Set("User-Agent", "Coraza+v3") - // TODO: declare content type in the formatter + req.Header.Set("Content-Type", h.formatter.MIME()) res, err := h.client.Do(req) if err != nil { return err diff --git a/internal/auditlog/https_writer_test.go b/internal/auditlog/https_writer_test.go index 900ce43b6..21c478d58 100644 --- a/internal/auditlog/https_writer_test.go +++ b/internal/auditlog/https_writer_test.go @@ -11,40 +11,45 @@ import ( "io" "net/http" "net/http/httptest" + "strings" "testing" "github.com/corazawaf/coraza/v3/experimental/plugins/plugintypes" "github.com/corazawaf/coraza/v3/types" ) +var sampleHttpsAuditLog = &Log{ + + Transaction_: Transaction{ + ID_: "test123", + }, + Messages_: []plugintypes.AuditLogMessage{ + Message{ + Data_: &MessageData{ + ID_: 100, + Raw_: "SecAction \"id:100\"", + }, + }, + }, +} + func TestHTTPSAuditLog(t *testing.T) { writer := &httpsWriter{} - formatter := nativeFormatter + formatter := &nativeFormatter{} pts, err := types.ParseAuditLogParts("ABCDEZ") if err != nil { t.Fatal(err) } - al := &Log{ - Parts_: pts, - - Transaction_: Transaction{ - ID_: "test123", - }, - Messages_: []plugintypes.AuditLogMessage{ - Message{ - Data_: &MessageData{ - ID_: 100, - Raw_: "SecAction \"id:100\"", - }, - }, - }, - } + sampleHttpsAuditLog.Parts_ = pts // we create a test http server server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) if r.ContentLength == 0 { t.Fatal("ContentLength is 0") } + if ct := r.Header.Get("Content-Type"); !strings.HasPrefix(ct, "application/x-coraza") { + t.Fatalf("Content-Type is not application/x-coraza, got %s", ct) + } // now we get the body body, err := io.ReadAll(r.Body) if err != nil { @@ -64,7 +69,32 @@ func TestHTTPSAuditLog(t *testing.T) { }); err != nil { t.Fatal(err) } - if err := writer.Write(al); err != nil { + if err := writer.Write(sampleHttpsAuditLog); err != nil { + t.Fatal(err) + } +} + +func TestJSONAuditHTTPS(t *testing.T) { + writer := &httpsWriter{} + formatter := &jsonFormatter{} + // we create a test http server + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + if r.ContentLength == 0 { + t.Fatal("ContentLength is 0") + } + if ct := r.Header.Get("Content-Type"); !strings.HasPrefix(ct, "application/json") { + t.Fatalf("Content-Type is not application/json, got %s", ct) + } + })) + defer server.Close() + if err := writer.Init(plugintypes.AuditLogConfig{ + Target: server.URL, + Formatter: formatter, + }); err != nil { + t.Fatal(err) + } + if err := writer.Write(sampleHttpsAuditLog); err != nil { t.Fatal(err) } } diff --git a/internal/auditlog/init.go b/internal/auditlog/init.go index cd45b9504..6984da43f 100644 --- a/internal/auditlog/init.go +++ b/internal/auditlog/init.go @@ -19,7 +19,7 @@ func init() { return &httpsWriter{} }) - RegisterFormatter("json", jsonFormatter) - RegisterFormatter("jsonlegacy", legacyJSONFormatter) - RegisterFormatter("native", nativeFormatter) + RegisterFormatter("json", &jsonFormatter{}) + RegisterFormatter("jsonlegacy", &legacyJSONFormatter{}) + RegisterFormatter("native", &nativeFormatter{}) } diff --git a/internal/auditlog/init_tinygo.go b/internal/auditlog/init_tinygo.go index 965865015..62ee561b8 100644 --- a/internal/auditlog/init_tinygo.go +++ b/internal/auditlog/init_tinygo.go @@ -20,7 +20,7 @@ func init() { }) // TODO(jcchavezs): check if newest TinyGo supports json.Marshaler for audit log type. - RegisterFormatter("json", noopFormater) - RegisterFormatter("jsonlegacy", noopFormater) - RegisterFormatter("native", nativeFormatter) + RegisterFormatter("json", &noopFormatter{}) + RegisterFormatter("jsonlegacy", &noopFormatter{}) + RegisterFormatter("native", &noopFormatter{}) } diff --git a/internal/auditlog/logger.go b/internal/auditlog/logger.go index 861a3764a..d3926bddb 100644 --- a/internal/auditlog/logger.go +++ b/internal/auditlog/logger.go @@ -17,7 +17,7 @@ func NewConfig() plugintypes.AuditLogConfig { FileMode: 0644, Dir: "", DirMode: 0755, - Formatter: nativeFormatter, + Formatter: &nativeFormatter{}, } } @@ -42,7 +42,7 @@ func GetWriter(name string) (plugintypes.AuditLogWriter, error) { // RegisterFormatter registers a new logger format // it can be used for plugins -func RegisterFormatter(name string, f func(plugintypes.AuditLog) ([]byte, error)) { +func RegisterFormatter(name string, f plugintypes.AuditLogFormatter) { formatters[name] = f } diff --git a/internal/auditlog/logger_test.go b/internal/auditlog/logger_test.go index 1c132a0f3..054e36f94 100644 --- a/internal/auditlog/logger_test.go +++ b/internal/auditlog/logger_test.go @@ -30,6 +30,11 @@ func TestGetUnknownWriter(t *testing.T) { } } +type noopFormatter struct{} + +func (noopFormatter) Format(al plugintypes.AuditLog) ([]byte, error) { return nil, nil } +func (noopFormatter) MIME() string { return "" } + func TestGetFormatters(t *testing.T) { t.Run("missing formatter", func(t *testing.T) { if _, err := GetFormatter("missing"); err == nil { @@ -38,14 +43,14 @@ func TestGetFormatters(t *testing.T) { }) t.Run("existing formatter", func(t *testing.T) { - expectedFn := func(al plugintypes.AuditLog) ([]byte, error) { return nil, nil } - RegisterFormatter("test", expectedFn) + f := &noopFormatter{} + RegisterFormatter("test", f) actualFn, err := GetFormatter("TeSt") if err != nil { t.Errorf("unexpected error: %s", err.Error()) } - if want, have := reflect.ValueOf(expectedFn), reflect.ValueOf(actualFn); want.Pointer() != have.Pointer() { + if want, have := reflect.ValueOf(f), reflect.ValueOf(actualFn); want.Pointer() != have.Pointer() { t.Errorf("unexpected formatter function") } }) diff --git a/internal/auditlog/noop_formater.go b/internal/auditlog/noop_formater.go index 95def7ff9..65211c4bd 100644 --- a/internal/auditlog/noop_formater.go +++ b/internal/auditlog/noop_formater.go @@ -9,6 +9,11 @@ package auditlog import "github.com/corazawaf/coraza/v3/experimental/plugins/plugintypes" -func noopFormater(plugintypes.AuditLog) ([]byte, error) { +type noopFormatter struct{} + +func (noopFormatter) Format(plugintypes.AuditLog) ([]byte, error) { return nil, nil } +func (noopFormatter) MIME() string { + return "" +} diff --git a/internal/auditlog/serial_writer.go b/internal/auditlog/serial_writer.go index f7782ed44..d752d541a 100644 --- a/internal/auditlog/serial_writer.go +++ b/internal/auditlog/serial_writer.go @@ -50,7 +50,7 @@ func (sl *serialWriter) Write(al plugintypes.AuditLog) error { return nil } - bts, err := sl.formatter(al) + bts, err := sl.formatter.Format(al) if err != nil { return err } diff --git a/internal/auditlog/serial_writer_test.go b/internal/auditlog/serial_writer_test.go index 4ef309326..72913e13b 100644 --- a/internal/auditlog/serial_writer_test.go +++ b/internal/auditlog/serial_writer_test.go @@ -61,7 +61,7 @@ func TestSerialWriterFailsOnInitForUnexistingFile(t *testing.T) { config.Dir = t.TempDir() config.FileMode = fs.FileMode(0777) config.DirMode = fs.FileMode(0777) - config.Formatter = jsonFormatter + config.Formatter = &jsonFormatter{} w := &serialWriter{} if err := w.Init(config); err == nil { @@ -78,7 +78,7 @@ func TestSerialWriterWrites(t *testing.T) { writer := &serialWriter{} config := NewConfig() config.Target = tmp - config.Formatter = jsonFormatter + config.Formatter = &jsonFormatter{} if err := writer.Init(config); err != nil { t.Fatal(err)