implement https mime (#850)

* implement https mime * refactor formatter interfaces * fix tinygo * Update experimental/plugins/auditlog_formatter_test.go Co-authored-by: José Carlos Chávez <jcchavezs@gmail.com> * minor lint * Update internal/auditlog/formats.go Co-authored-by: José Carlos Chávez <jcchavezs@gmail.com> * minor lint * Update experimental/plugins/plugintypes/auditlog.go Co-authored-by: José Carlos Chávez <jcchavezs@gmail.com> * Update internal/auditlog/formats_json.go Co-authored-by: José Carlos Chávez <jcchavezs@gmail.com> * Update internal/auditlog/formats_json.go Co-authored-by: José Carlos Chávez <jcchavezs@gmail.com> * add more tests * fix test * fix https formatter * remove factory for formatter --------- Co-authored-by: José Carlos Chávez <jcchavezs@gmail.com>
corazawaf · Aug 6, 2023 · b3490b4 · b3490b4
1 parent cecd79c
commit b3490b4
Show file tree

Hide file tree

Showing 18 changed files with 133 additions and 54 deletions.
diff --git a/experimental/plugins/auditlog.go b/experimental/plugins/auditlog.go
@@ -14,6 +14,6 @@ func RegisterAuditLogWriter(name string, writerFactory func() plugintypes.AuditL
 }
 
 // RegisterAuditLogFormatter registers a new audit log formatter.
-func RegisterAuditLogFormatter(name string, f func(plugintypes.AuditLog) ([]byte, error)) {
-	auditlog.RegisterFormatter(name, f)
+func RegisterAuditLogFormatter(name string, format plugintypes.AuditLogFormatter) {
+	auditlog.RegisterFormatter(name, format)
 }
diff --git a/experimental/plugins/auditlog_formatter_test.go b/experimental/plugins/auditlog_formatter_test.go
@@ -14,12 +14,21 @@ import (
 	"github.com/corazawaf/coraza/v3/experimental/plugins/plugintypes"
 )
 
+type testFormatter struct{}
+
+func (testFormatter) Format(al plugintypes.AuditLog) ([]byte, error) {
+	return []byte(al.Transaction().ID()), nil
+}
+
+func (testFormatter) MIME() string {
+	return "sample"
+}
+
 // ExampleRegisterAuditLogFormatter shows how to register a custom audit log formatter
 // and tests the output of the formatter.
 func ExampleRegisterAuditLogFormatter() {
-	plugins.RegisterAuditLogFormatter("txid", func(al plugintypes.AuditLog) ([]byte, error) {
-		return []byte(al.Transaction().ID()), nil
-	})
+
+	plugins.RegisterAuditLogFormatter("txid", &testFormatter{})
 
 	w, err := coraza.NewWAF(
 		coraza.NewWAFConfig().

diff --git a/experimental/plugins/plugintypes/auditlog.go b/experimental/plugins/plugintypes/auditlog.go
@@ -128,5 +128,9 @@ type AuditLogWriter interface {
 	Close() error
 }
 
-// AuditLogFormatter formats an audit log to a byte slice.
-type AuditLogFormatter func(AuditLog) ([]byte, error)
+// AuditLogFormatter serializes an AuditLog into a byte slice.
+// It is used to construct the formatted audit log.
+type AuditLogFormatter interface {
+	Format(AuditLog) ([]byte, error)
+	MIME() string
+}
diff --git a/internal/auditlog/concurrent_writer.go b/internal/auditlog/concurrent_writer.go
@@ -67,7 +67,7 @@ func (cl concurrentWriter) Write(al plugintypes.AuditLog) error {
 		return err
 	}
 
-	formattedAL, err := cl.formatter(al)
+	formattedAL, err := cl.formatter.Format(al)
 	if err != nil {
 		return err
 	}

diff --git a/internal/auditlog/concurrent_writer_test.go b/internal/auditlog/concurrent_writer_test.go
@@ -38,7 +38,7 @@ func TestConcurrentWriterFailsOnInit(t *testing.T) {
 	config.Dir = t.TempDir()
 	config.FileMode = fs.FileMode(0777)
 	config.DirMode = fs.FileMode(0777)
-	config.Formatter = jsonFormatter
+	config.Formatter = &jsonFormatter{}
 
 	writer := &concurrentWriter{}
 	if err := writer.Init(config); err == nil {
@@ -57,7 +57,7 @@ func TestConcurrentWriterWrites(t *testing.T) {
 		Dir:       dir,
 		FileMode:  fs.FileMode(0777),
 		DirMode:   fs.FileMode(0777),
-		Formatter: jsonFormatter,
+		Formatter: &jsonFormatter{},
 	}
 	ts := time.Now()
 	expectedLog := &Log{

diff --git a/internal/auditlog/formats.go b/internal/auditlog/formats.go
@@ -28,7 +28,9 @@ import (
 	"github.com/corazawaf/coraza/v3/types"
 )
 
-func nativeFormatter(al plugintypes.AuditLog) ([]byte, error) {
+type nativeFormatter struct{}
+
+func (nativeFormatter) Format(al plugintypes.AuditLog) ([]byte, error) {
 	boundaryPrefix := fmt.Sprintf("--%s-", utils.RandomString(10))
 
 	var res strings.Builder
@@ -102,6 +104,10 @@ func nativeFormatter(al plugintypes.AuditLog) ([]byte, error) {
 	return []byte(res.String()), nil
 }
 
+func (nativeFormatter) MIME() string {
+	return "application/x-coraza-auditlog-native"
+}
+
 var (
-	_ plugintypes.AuditLogFormatter = nativeFormatter
+	_ plugintypes.AuditLogFormatter = (*nativeFormatter)(nil)
 )
diff --git a/internal/auditlog/formats_json.go b/internal/auditlog/formats_json.go
@@ -15,17 +15,23 @@ import (
 	"github.com/corazawaf/coraza/v3/experimental/plugins/plugintypes"
 )
 
-// Coraza format
-func jsonFormatter(al plugintypes.AuditLog) ([]byte, error) {
+type jsonFormatter struct{}
+
+func (jsonFormatter) Format(al plugintypes.AuditLog) ([]byte, error) {
 	jsdata, err := json.Marshal(al)
 	if err != nil {
 		return nil, err
 	}
 	return jsdata, nil
 }
 
-// Coraza legacy json format
-func legacyJSONFormatter(al plugintypes.AuditLog) ([]byte, error) {
+func (jsonFormatter) MIME() string {
+	return "application/json; charset=utf-8"
+}
+
+type legacyJSONFormatter struct{}
+
+func (_ legacyJSONFormatter) Format(al plugintypes.AuditLog) ([]byte, error) {
 	al2 := logLegacy{
 		Transaction: logLegacyTransaction{
 			Time:          al.Transaction().Timestamp(),
@@ -92,7 +98,11 @@ func legacyJSONFormatter(al plugintypes.AuditLog) ([]byte, error) {
 	return jsdata, nil
 }
 
+func (_ legacyJSONFormatter) MIME() string {
+	return "application/json; charset=utf-8"
+}
+
 var (
-	_ plugintypes.AuditLogFormatter = jsonFormatter
-	_ plugintypes.AuditLogFormatter = legacyJSONFormatter
+	_ plugintypes.AuditLogFormatter = (*jsonFormatter)(nil)
+	_ plugintypes.AuditLogFormatter = (*legacyJSONFormatter)(nil)
 )
diff --git a/internal/auditlog/formats_json_test.go b/internal/auditlog/formats_json_test.go
@@ -8,6 +8,7 @@ package auditlog
 
 import (
 	"encoding/json"
+	"strings"
 	"testing"
 )
 
@@ -48,10 +49,14 @@ func TestModsecBoundary(t *testing.T) {
 
 func TestLegacyFormatter(t *testing.T) {
 	al := createAuditLog()
-	data, err := legacyJSONFormatter(al)
+	f := &legacyJSONFormatter{}
+	data, err := f.Format(al)
 	if err != nil {
 		t.Error(err)
 	}
+	if !strings.Contains(f.MIME(), "json") {
+		t.Errorf("failed to match MIME, expected json and got %s", f.MIME())
+	}
 	var legacyAl logLegacy
 	if err := json.Unmarshal(data, &legacyAl); err != nil {
 		t.Error(err)

diff --git a/internal/auditlog/formats_test.go b/internal/auditlog/formats_test.go
@@ -5,6 +5,7 @@ package auditlog
 
 import (
 	"bytes"
+	"strings"
 	"testing"
 
 	"github.com/corazawaf/coraza/v3/experimental/plugins/plugintypes"
@@ -13,10 +14,14 @@ import (
 
 func TestNativeFormatter(t *testing.T) {
 	al := createAuditLog()
-	data, err := nativeFormatter(al)
+	f := &nativeFormatter{}
+	data, err := f.Format(al)
 	if err != nil {
 		t.Error(err)
 	}
+	if !strings.Contains(f.MIME(), "x-coraza-auditlog-native") {
+		t.Errorf("failed to match MIME, expected json and got %s", f.MIME())
+	}
 	// Log contains random strings, do a simple sanity check
 	if !bytes.Contains(data, []byte("[02/Jan/2006:15:04:20 -0700] 123  0  0")) {
 		t.Errorf("failed to match log, \ngot: %s\n", string(data))

diff --git a/internal/auditlog/https_writer.go b/internal/auditlog/https_writer.go
@@ -41,7 +41,7 @@ func (h *httpsWriter) Init(c plugintypes.AuditLogConfig) error {
 }
 
 func (h *httpsWriter) Write(al plugintypes.AuditLog) error {
-	body, err := h.formatter(al)
+	body, err := h.formatter.Format(al)
 	if err != nil {
 		return err
 	}
@@ -51,7 +51,7 @@ func (h *httpsWriter) Write(al plugintypes.AuditLog) error {
 		return err
 	}
 	req.Header.Set("User-Agent", "Coraza+v3")
-	// TODO: declare content type in the formatter
+	req.Header.Set("Content-Type", h.formatter.MIME())
 	res, err := h.client.Do(req)
 	if err != nil {
 		return err

diff --git a/internal/auditlog/https_writer_test.go b/internal/auditlog/https_writer_test.go
@@ -11,40 +11,45 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"testing"
 
 	"github.com/corazawaf/coraza/v3/experimental/plugins/plugintypes"
 	"github.com/corazawaf/coraza/v3/types"
 )
 
+var sampleHttpsAuditLog = &Log{
+
+	Transaction_: Transaction{
+		ID_: "test123",
+	},
+	Messages_: []plugintypes.AuditLogMessage{
+		Message{
+			Data_: &MessageData{
+				ID_:  100,
+				Raw_: "SecAction \"id:100\"",
+			},
+		},
+	},
+}
+
 func TestHTTPSAuditLog(t *testing.T) {
 	writer := &httpsWriter{}
-	formatter := nativeFormatter
+	formatter := &nativeFormatter{}
 	pts, err := types.ParseAuditLogParts("ABCDEZ")
 	if err != nil {
 		t.Fatal(err)
 	}
-	al := &Log{
-		Parts_: pts,
-
-		Transaction_: Transaction{
-			ID_: "test123",
-		},
-		Messages_: []plugintypes.AuditLogMessage{
-			Message{
-				Data_: &MessageData{
-					ID_:  100,
-					Raw_: "SecAction \"id:100\"",
-				},
-			},
-		},
-	}
+	sampleHttpsAuditLog.Parts_ = pts
 	// we create a test http server
 	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		w.WriteHeader(http.StatusOK)
 		if r.ContentLength == 0 {
 			t.Fatal("ContentLength is 0")
 		}
+		if ct := r.Header.Get("Content-Type"); !strings.HasPrefix(ct, "application/x-coraza") {
+			t.Fatalf("Content-Type is not application/x-coraza, got %s", ct)
+		}
 		// now we get the body
 		body, err := io.ReadAll(r.Body)
 		if err != nil {
@@ -64,7 +69,32 @@ func TestHTTPSAuditLog(t *testing.T) {
 	}); err != nil {
 		t.Fatal(err)
 	}
-	if err := writer.Write(al); err != nil {
+	if err := writer.Write(sampleHttpsAuditLog); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestJSONAuditHTTPS(t *testing.T) {
+	writer := &httpsWriter{}
+	formatter := &jsonFormatter{}
+	// we create a test http server
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		if r.ContentLength == 0 {
+			t.Fatal("ContentLength is 0")
+		}
+		if ct := r.Header.Get("Content-Type"); !strings.HasPrefix(ct, "application/json") {
+			t.Fatalf("Content-Type is not application/json, got %s", ct)
+		}
+	}))
+	defer server.Close()
+	if err := writer.Init(plugintypes.AuditLogConfig{
+		Target:    server.URL,
+		Formatter: formatter,
+	}); err != nil {
+		t.Fatal(err)
+	}
+	if err := writer.Write(sampleHttpsAuditLog); err != nil {
 		t.Fatal(err)
 	}
 }
diff --git a/internal/auditlog/init.go b/internal/auditlog/init.go
@@ -19,7 +19,7 @@ func init() {
 		return &httpsWriter{}
 	})
 
-	RegisterFormatter("json", jsonFormatter)
-	RegisterFormatter("jsonlegacy", legacyJSONFormatter)
-	RegisterFormatter("native", nativeFormatter)
+	RegisterFormatter("json", &jsonFormatter{})
+	RegisterFormatter("jsonlegacy", &legacyJSONFormatter{})
+	RegisterFormatter("native", &nativeFormatter{})
 }
diff --git a/internal/auditlog/init_tinygo.go b/internal/auditlog/init_tinygo.go
@@ -20,7 +20,7 @@ func init() {
 	})
 
 	// TODO(jcchavezs): check if newest TinyGo supports json.Marshaler for audit log type.
-	RegisterFormatter("json", noopFormater)
-	RegisterFormatter("jsonlegacy", noopFormater)
-	RegisterFormatter("native", nativeFormatter)
+	RegisterFormatter("json", &noopFormatter{})
+	RegisterFormatter("jsonlegacy", &noopFormatter{})
+	RegisterFormatter("native", &noopFormatter{})
 }
diff --git a/internal/auditlog/logger.go b/internal/auditlog/logger.go
@@ -17,7 +17,7 @@ func NewConfig() plugintypes.AuditLogConfig {
 		FileMode:  0644,
 		Dir:       "",
 		DirMode:   0755,
-		Formatter: nativeFormatter,
+		Formatter: &nativeFormatter{},
 	}
 }
 
@@ -42,7 +42,7 @@ func GetWriter(name string) (plugintypes.AuditLogWriter, error) {
 
 // RegisterFormatter registers a new logger format
 // it can be used for plugins
-func RegisterFormatter(name string, f func(plugintypes.AuditLog) ([]byte, error)) {
+func RegisterFormatter(name string, f plugintypes.AuditLogFormatter) {
 	formatters[name] = f
 }
 

diff --git a/internal/auditlog/logger_test.go b/internal/auditlog/logger_test.go
@@ -30,6 +30,11 @@ func TestGetUnknownWriter(t *testing.T) {
 	}
 }
 
+type noopFormatter struct{}
+
+func (noopFormatter) Format(al plugintypes.AuditLog) ([]byte, error) { return nil, nil }
+func (noopFormatter) MIME() string                                   { return "" }
+
 func TestGetFormatters(t *testing.T) {
 	t.Run("missing formatter", func(t *testing.T) {
 		if _, err := GetFormatter("missing"); err == nil {
@@ -38,14 +43,14 @@ func TestGetFormatters(t *testing.T) {
 	})
 
 	t.Run("existing formatter", func(t *testing.T) {
-		expectedFn := func(al plugintypes.AuditLog) ([]byte, error) { return nil, nil }
-		RegisterFormatter("test", expectedFn)
+		f := &noopFormatter{}
+		RegisterFormatter("test", f)
 		actualFn, err := GetFormatter("TeSt")
 		if err != nil {
 			t.Errorf("unexpected error: %s", err.Error())
 		}
 
-		if want, have := reflect.ValueOf(expectedFn), reflect.ValueOf(actualFn); want.Pointer() != have.Pointer() {
+		if want, have := reflect.ValueOf(f), reflect.ValueOf(actualFn); want.Pointer() != have.Pointer() {
 			t.Errorf("unexpected formatter function")
 		}
 	})

diff --git a/internal/auditlog/noop_formater.go b/internal/auditlog/noop_formater.go
@@ -9,6 +9,11 @@ package auditlog
 
 import "github.com/corazawaf/coraza/v3/experimental/plugins/plugintypes"
 
-func noopFormater(plugintypes.AuditLog) ([]byte, error) {
+type noopFormatter struct{}
+
+func (noopFormatter) Format(plugintypes.AuditLog) ([]byte, error) {
 	return nil, nil
 }
+func (noopFormatter) MIME() string {
+	return ""
+}
diff --git a/internal/auditlog/serial_writer.go b/internal/auditlog/serial_writer.go
@@ -50,7 +50,7 @@ func (sl *serialWriter) Write(al plugintypes.AuditLog) error {
 		return nil
 	}
 
-	bts, err := sl.formatter(al)
+	bts, err := sl.formatter.Format(al)
 	if err != nil {
 		return err
 	}