From 65bc7d3642c599322ef4e8688fe8259dd3191ea6 Mon Sep 17 00:00:00 2001
From: Bremer Jonathan <bjonathan@splunk.com>
Date: Mon, 1 Mar 2021 04:16:44 -0500
Subject: [PATCH 01/26]  limit splunkhecexporter request content length for
 logs

---
 exporter/splunkhecexporter/client.go          |  52 ++++++-
 exporter/splunkhecexporter/config.go          |   3 +
 exporter/splunkhecexporter/config_test.go     |  13 +-
 exporter/splunkhecexporter/factory.go         |   8 +-
 .../splunkhecexporter/logdata_to_splunk.go    | 147 ++++++++++++++++--
 .../logdata_to_splunk_test.go                 | 101 ++++++++----
 6 files changed, 264 insertions(+), 60 deletions(-)

diff --git a/exporter/splunkhecexporter/client.go b/exporter/splunkhecexporter/client.go
index 602c9d2dab9a..b6182caa8433 100644
--- a/exporter/splunkhecexporter/client.go
+++ b/exporter/splunkhecexporter/client.go
@@ -121,7 +121,11 @@ func (c *client) sendSplunkEvents(ctx context.Context, splunkEvents []*splunk.Ev
 		return consumererror.Permanent(err)
 	}
 
-	req, err := http.NewRequestWithContext(ctx, "POST", c.url.String(), body)
+	return c.postEvents(ctx, body, compressed)
+}
+
+func (c *client) postEvents(ctx context.Context, events io.Reader, compressed bool) error {
+	req, err := http.NewRequestWithContext(ctx, "POST", c.url.String(), events)
 	if err != nil {
 		return consumererror.Permanent(err)
 	}
@@ -157,14 +161,46 @@ func (c *client) pushLogData(ctx context.Context, ld pdata.Logs) (numDroppedLogs
 	c.wg.Add(1)
 	defer c.wg.Done()
 
-	splunkEvents := logDataToSplunk(c.logger, ld, c.config)
-	if len(splunkEvents) == 0 {
-		return 0, nil
-	}
+	gwriter := c.zippers.Get().(*gzip.Writer)
+	defer c.zippers.Put(gwriter)
 
-	err = c.sendSplunkEvents(ctx, splunkEvents)
-	if err != nil {
-		return ld.LogRecordCount(), err
+	gzipBuf := bytes.NewBuffer(make([]byte, 0, c.config.MaxContentLength))
+	gwriter.Reset(gzipBuf)
+	defer gwriter.Close()
+
+	ldWrapper := logDataWrapper{&ld}
+	eventsCh, cancel := ldWrapper.eventsInChunks(c.logger, c.config)
+	defer cancel()
+
+	for events := range eventsCh {
+		if events.err != nil {
+			return ldWrapper.processErr(events.index, events.err)
+		}
+
+		if events.buf.Len() == 0 {
+			continue
+		}
+
+		// Not compressing if compression disabled or payload fit into a single ethernet frame.
+		if events.buf.Len() <= 1500 || c.config.DisableCompression {
+			if err = c.postEvents(ctx, events.buf, false); err != nil {
+				return ldWrapper.processErr(events.index, err)
+			}
+			continue
+		}
+
+		if _, err = gwriter.Write(events.buf.Bytes()); err != nil {
+			return ldWrapper.processErr(events.index, consumererror.Permanent(err))
+		}
+
+		gwriter.Flush()
+
+		if err = c.postEvents(ctx, gzipBuf, true); err != nil {
+			return ldWrapper.processErr(events.index, err)
+		}
+
+		gzipBuf.Reset()
+		gwriter.Reset(gzipBuf)
 	}
 
 	return 0, nil
diff --git a/exporter/splunkhecexporter/config.go b/exporter/splunkhecexporter/config.go
index 8bf3b8b6c5fc..1da8e1f3bf9e 100644
--- a/exporter/splunkhecexporter/config.go
+++ b/exporter/splunkhecexporter/config.go
@@ -60,6 +60,9 @@ type Config struct {
 
 	// insecure_skip_verify skips checking the certificate of the HEC endpoint when sending data over HTTPS. Defaults to false.
 	InsecureSkipVerify bool `mapstructure:"insecure_skip_verify"`
+
+	// MaxContentLength is the Splunk HEC endpoint content length limit. Defaults to 1Mib, the current limit.
+	MaxContentLength int `mapstructure:"max_content_length"`
 }
 
 func (cfg *Config) getOptionsFromConfig() (*exporterOptions, error) {
diff --git a/exporter/splunkhecexporter/config_test.go b/exporter/splunkhecexporter/config_test.go
index 85c04008b993..29f17f0292e2 100644
--- a/exporter/splunkhecexporter/config_test.go
+++ b/exporter/splunkhecexporter/config_test.go
@@ -58,12 +58,13 @@ func TestLoadConfig(t *testing.T) {
 			TypeVal: configmodels.Type(typeStr),
 			NameVal: expectedName,
 		},
-		Token:          "00000000-0000-0000-0000-0000000000000",
-		Endpoint:       "https://splunk:8088/services/collector",
-		Source:         "otel",
-		SourceType:     "otel",
-		Index:          "metrics",
-		MaxConnections: 100,
+		Token:            "00000000-0000-0000-0000-0000000000000",
+		Endpoint:         "https://splunk:8088/services/collector",
+		Source:           "otel",
+		SourceType:       "otel",
+		Index:            "metrics",
+		MaxConnections:   100,
+		MaxContentLength: 1024 * 1024,
 		TimeoutSettings: exporterhelper.TimeoutSettings{
 			Timeout: 10 * time.Second,
 		},
diff --git a/exporter/splunkhecexporter/factory.go b/exporter/splunkhecexporter/factory.go
index 6de32ca1fdc5..6b1c1f29ee9c 100644
--- a/exporter/splunkhecexporter/factory.go
+++ b/exporter/splunkhecexporter/factory.go
@@ -26,9 +26,10 @@ import (
 
 const (
 	// The value of "type" key in configuration.
-	typeStr            = "splunk_hec"
-	defaultMaxIdleCons = 100
-	defaultHTTPTimeout = 10 * time.Second
+	typeStr                 = "splunk_hec"
+	defaultMaxIdleCons      = 100
+	defaultHTTPTimeout      = 10 * time.Second
+	defaultMaxContentLength = 1024 * 1024
 )
 
 // NewFactory creates a factory for Splunk HEC exporter.
@@ -54,6 +55,7 @@ func createDefaultConfig() configmodels.Exporter {
 		QueueSettings:      exporterhelper.DefaultQueueSettings(),
 		DisableCompression: false,
 		MaxConnections:     defaultMaxIdleCons,
+		MaxContentLength:   defaultMaxContentLength,
 	}
 }
 
diff --git a/exporter/splunkhecexporter/logdata_to_splunk.go b/exporter/splunkhecexporter/logdata_to_splunk.go
index 1b80e4a56562..d5113d8963a1 100644
--- a/exporter/splunkhecexporter/logdata_to_splunk.go
+++ b/exporter/splunkhecexporter/logdata_to_splunk.go
@@ -15,8 +15,13 @@
 package splunkhecexporter
 
 import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
 	"time"
 
+	"go.opentelemetry.io/collector/consumer/consumererror"
 	"go.opentelemetry.io/collector/consumer/pdata"
 	"go.opentelemetry.io/collector/translator/conventions"
 	"go.uber.org/zap"
@@ -24,20 +29,142 @@ import (
 	"github.com/open-telemetry/opentelemetry-collector-contrib/internal/splunk"
 )
 
-func logDataToSplunk(logger *zap.Logger, ld pdata.Logs, config *Config) []*splunk.Event {
-	var splunkEvents []*splunk.Event
-	rls := ld.ResourceLogs()
-	for i := 0; i < rls.Len(); i++ {
-		ills := rls.At(i).InstrumentationLibraryLogs()
-		for j := 0; j < ills.Len(); j++ {
-			logs := ills.At(j).Logs()
-			for k := 0; k < logs.Len(); k++ {
-				splunkEvents = append(splunkEvents, mapLogRecordToSplunkEvent(logs.At(k), config, logger))
+// eventsBuf is a buffer of JSON encoded Splunk events.
+// The events are created from LogRecord(s) where one event maps to one LogRecord.
+type eventsBuf struct {
+	buf *bytes.Buffer
+	// index is the eventIndex of the 1st event in buf.
+	index *eventIndex
+	err   error
+}
+
+// The index of an event composed of indices of the event's LogRecord.
+type eventIndex struct {
+	// Index of the LogRecord slice element from which the event is created.
+	log int
+	// Index of the InstrumentationLibraryLogs slice element parent of the LogRecord.
+	lib int
+	// Index of the ResourceLogs slice element parent of the InstrumentationLibraryLogs.
+	src int
+}
+
+type logDataWrapper struct {
+	*pdata.Logs
+}
+
+func (ld *logDataWrapper) eventsInChunks(logger *zap.Logger, config *Config) (chan *eventsBuf, context.CancelFunc) {
+	ctx, cancel := context.WithCancel(context.Background())
+	eventsCh := make(chan *eventsBuf)
+
+	go func() {
+		defer close(eventsCh)
+
+		// event buffers a single event.
+		event := new(bytes.Buffer)
+		encoder := json.NewEncoder(event)
+
+		// events buffers events up to the max content length.
+		events := &eventsBuf{buf: new(bytes.Buffer)}
+
+		rl := ld.ResourceLogs()
+		for i := 0; i < rl.Len(); i++ {
+			ill := rl.At(i).InstrumentationLibraryLogs()
+			for j := 0; j < ill.Len(); j++ {
+				l := ill.At(j).Logs()
+				for k := 0; k < l.Len(); k++ {
+					select {
+					case <-ctx.Done():
+						return
+					default:
+						if err := encoder.Encode(mapLogRecordToSplunkEvent(l.At(k), config, logger)); err != nil {
+							eventsCh <- &eventsBuf{buf: nil, index: nil, err: consumererror.Permanent(err)}
+							return
+						}
+						event.WriteString("\r\n\r\n")
+
+						// The size of an event must be less than or equal to max content length.
+						if config.MaxContentLength > 0 && event.Len() > config.MaxContentLength {
+							err := fmt.Errorf("found a log event bigger than max content length (event: %d bytes, max: %d bytes)", config.MaxContentLength, event.Len())
+							eventsCh <- &eventsBuf{buf: nil, index: nil, err: consumererror.Permanent(err)}
+							return
+						}
+
+						// Moving the event to events.buf if length will be <= max content length.
+						// Max content length <= 0 is interpreted as unbound.
+						if events.buf.Len()+event.Len() <= config.MaxContentLength || config.MaxContentLength <= 0 {
+							// WriteTo() empties and resets buffer event.
+							if _, err := event.WriteTo(events.buf); err != nil {
+								eventsCh <- &eventsBuf{buf: nil, index: nil, err: consumererror.Permanent(err)}
+								return
+							}
+
+							// Setting events index using the log record indices of the 1st event.
+							if events.index == nil {
+								events.index = &eventIndex{src: i, lib: j, log: k}
+							}
+
+							continue
+						}
+
+						eventsCh <- events
+
+						// Creating a new events buffer.
+						events = &eventsBuf{buf: new(bytes.Buffer)}
+						// Setting events index using the log record indices of any current leftover event.
+						if event.Len() != 0 {
+							events.index = &eventIndex{src: i, lib: j, log: k}
+						}
+					}
+				}
+			}
+		}
+
+		// Writing any leftover event to eventsBuf buffer `events.buf`.
+		if _, err := event.WriteTo(events.buf); err != nil {
+			eventsCh <- &eventsBuf{buf: nil, index: nil, err: consumererror.Permanent(err)}
+			return
+		}
+
+		eventsCh <- events
+
+	}()
+	return eventsCh, cancel
+}
+
+func (ld *logDataWrapper) countLogs(start *eventIndex) int {
+	count, orig := 0, *ld.InternalRep().Orig
+	for i := start.src; i < len(orig); i++ {
+		for j, iLLogs := range orig[i].InstrumentationLibraryLogs {
+			switch {
+			case i == start.src && j < start.lib:
+				continue
+			default:
+				count += len(iLLogs.Logs)
 			}
 		}
 	}
+	return count - start.log
+}
+
+func (ld *logDataWrapper) trimLeft(end *eventIndex) *pdata.Logs {
+	clone := ld.Clone()
+	orig := *clone.InternalRep().Orig
+	orig = orig[end.src:]
+	orig[end.src].InstrumentationLibraryLogs = orig[end.src].InstrumentationLibraryLogs[end.lib:]
+	orig[end.src].InstrumentationLibraryLogs[end.lib].Logs = orig[end.src].InstrumentationLibraryLogs[end.lib].Logs[end.log:]
+	return &clone
+}
 
-	return splunkEvents
+func (ld *logDataWrapper) processErr(index *eventIndex, err error) (int, error) {
+	if consumererror.IsPermanent(err) {
+		return ld.countLogs(index), err
+	}
+
+	if _, ok := err.(consumererror.PartialError); ok {
+		failedLogs := ld.trimLeft(index)
+		return failedLogs.LogRecordCount(), consumererror.PartialLogsError(err, *failedLogs)
+	}
+	return ld.LogRecordCount(), err
 }
 
 func mapLogRecordToSplunkEvent(lr pdata.LogRecord, config *Config, logger *zap.Logger) *splunk.Event {
diff --git a/exporter/splunkhecexporter/logdata_to_splunk_test.go b/exporter/splunkhecexporter/logdata_to_splunk_test.go
index 7d28374dbbbb..14059fdc186f 100644
--- a/exporter/splunkhecexporter/logdata_to_splunk_test.go
+++ b/exporter/splunkhecexporter/logdata_to_splunk_test.go
@@ -15,6 +15,8 @@
 package splunkhecexporter
 
 import (
+	"bytes"
+	"encoding/json"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -112,7 +114,7 @@ func Test_logDataToSplunk(t *testing.T) {
 				}
 			},
 			wantSplunkEvents: []*splunk.Event{
-				commonLogSplunkEvent(nil, 0, map[string]interface{}{}, "unknown", "source", "sourcetype"),
+				commonLogSplunkEvent(nil, 0, nil, "unknown", "source", "sourcetype"),
 			},
 		},
 		{
@@ -137,28 +139,29 @@ func Test_logDataToSplunk(t *testing.T) {
 				commonLogSplunkEvent(float64(42), ts, map[string]interface{}{"custom": "custom"}, "myhost", "myapp", "myapp-type"),
 			},
 		},
-		{
-			name: "with int body",
-			logDataFn: func() pdata.Logs {
-				logRecord := pdata.NewLogRecord()
-				logRecord.Body().SetIntVal(42)
-				logRecord.Attributes().InsertString(conventions.AttributeServiceName, "myapp")
-				logRecord.Attributes().InsertString(splunk.SourcetypeLabel, "myapp-type")
-				logRecord.Attributes().InsertString(conventions.AttributeHostName, "myhost")
-				logRecord.Attributes().InsertString("custom", "custom")
-				logRecord.SetTimestamp(ts)
-				return makeLog(logRecord)
-			},
-			configDataFn: func() *Config {
-				return &Config{
-					Source:     "source",
-					SourceType: "sourcetype",
-				}
-			},
-			wantSplunkEvents: []*splunk.Event{
-				commonLogSplunkEvent(int64(42), ts, map[string]interface{}{"custom": "custom"}, "myhost", "myapp", "myapp-type"),
-			},
-		},
+		// TODO: int64 body gets unmarshalled to float64 because splunk.Event.Event is of type interface{}
+		//{
+		//	name: "with int body",
+		//	logDataFn: func() pdata.Logs {
+		//		logRecord := pdata.NewLogRecord()
+		//		logRecord.Body().SetIntVal(42)
+		//		logRecord.Attributes().InsertString(conventions.AttributeServiceName, "myapp")
+		//		logRecord.Attributes().InsertString(splunk.SourcetypeLabel, "myapp-type")
+		//		logRecord.Attributes().InsertString(conventions.AttributeHostName, "myhost")
+		//		logRecord.Attributes().InsertString("custom", "custom")
+		//		logRecord.SetTimestamp(ts)
+		//		return makeLog(logRecord)
+		//	},
+		//	configDataFn: func() *Config {
+		//		return &Config{
+		//			Source:     "source",
+		//			SourceType: "sourcetype",
+		//		}
+		//	},
+		//	wantSplunkEvents: []*splunk.Event{
+		//		commonLogSplunkEvent(int64(42), ts, map[string]interface{}{"custom": "custom"}, "myhost", "myapp", "myapp-type"),
+		//	},
+		//},
 		{
 			name: "with bool body",
 			logDataFn: func() pdata.Logs {
@@ -256,10 +259,23 @@ func Test_logDataToSplunk(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			gotEvents := logDataToSplunk(logger, tt.logDataFn(), tt.configDataFn())
-			require.Equal(t, len(tt.wantSplunkEvents), len(gotEvents))
-			for i, want := range tt.wantSplunkEvents {
-				assert.EqualValues(t, want, gotEvents[i])
+			logs := tt.logDataFn()
+			logsWrapper := logDataWrapper{&logs}
+
+			ch, cancel := logsWrapper.eventsInChunks(logger, tt.configDataFn())
+			defer cancel()
+
+			events := bytes.Split(bytes.TrimSpace((<-ch).buf.Bytes()), []byte("\r\n\r\n"))
+
+			require.Equal(t, len(tt.wantSplunkEvents), len(events))
+
+			var gotEvent splunk.Event
+			var gotEvents []*splunk.Event
+
+			for i, event := range events {
+				json.Unmarshal(event, &gotEvent)
+				assert.EqualValues(t, tt.wantSplunkEvents[i], &gotEvent)
+				gotEvents = append(gotEvents, &gotEvent)
 			}
 			assert.Equal(t, tt.wantSplunkEvents, gotEvents)
 		})
@@ -295,24 +311,43 @@ func commonLogSplunkEvent(
 }
 
 func Test_nilLogs(t *testing.T) {
-	events := logDataToSplunk(zap.NewNop(), pdata.NewLogs(), &Config{})
-	assert.Equal(t, 0, len(events))
+	//events := logDataToSplunk(zap.NewNop(), pdata.NewLogs(), &Config{})
+	//assert.Equal(t, 0, len(events))
+	logs := pdata.NewLogs()
+	ldWrap := logDataWrapper{&logs}
+	eventsCh, cancel := ldWrap.eventsInChunks(zap.NewNop(), &Config{})
+	defer cancel()
+	events := <-eventsCh
+	assert.Equal(t, 0, events.buf.Len())
 }
 
 func Test_nilResourceLogs(t *testing.T) {
 	logs := pdata.NewLogs()
 	logs.ResourceLogs().Resize(1)
-	events := logDataToSplunk(zap.NewNop(), logs, &Config{})
-	assert.Equal(t, 0, len(events))
+	ldWrap := logDataWrapper{&logs}
+	//events := logDataToSplunk(zap.NewNop(), logs, &Config{})
+	eventsCh, cancel := ldWrap.eventsInChunks(zap.NewNop(), &Config{})
+	defer cancel()
+	events := <-eventsCh
+	assert.Equal(t, 0, events.buf.Len())
 }
 
 func Test_nilInstrumentationLogs(t *testing.T) {
+	//logs := pdata.NewLogs()
+	//logs.ResourceLogs().Resize(1)
+	//resourceLog := logs.ResourceLogs().At(0)
+	//resourceLog.InstrumentationLibraryLogs().Resize(1)
+	//events := logDataToSplunk(zap.NewNop(), logs, &Config{})
+	//assert.Equal(t, 0, len(events))
 	logs := pdata.NewLogs()
 	logs.ResourceLogs().Resize(1)
 	resourceLog := logs.ResourceLogs().At(0)
 	resourceLog.InstrumentationLibraryLogs().Resize(1)
-	events := logDataToSplunk(zap.NewNop(), logs, &Config{})
-	assert.Equal(t, 0, len(events))
+	ldWrap := logDataWrapper{&logs}
+	eventsCh, cancel := ldWrap.eventsInChunks(zap.NewNop(), &Config{})
+	defer cancel()
+	events := <-eventsCh
+	assert.Equal(t, 0, events.buf.Len())
 }
 
 func Test_nanoTimestampToEpochMilliseconds(t *testing.T) {

From c66dee4bb532d165642d1dec586179dff1bea141 Mon Sep 17 00:00:00 2001
From: Bremer Jonathan <bjonathan@splunk.com>
Date: Mon, 1 Mar 2021 23:43:13 -0500
Subject: [PATCH 02/26] add unit tests

---
 exporter/splunkhecexporter/client.go          |  26 +--
 .../splunkhecexporter/logdata_to_splunk.go    |  71 ++++---
 .../logdata_to_splunk_test.go                 | 177 ++++++++++++++----
 3 files changed, 187 insertions(+), 87 deletions(-)

diff --git a/exporter/splunkhecexporter/client.go b/exporter/splunkhecexporter/client.go
index b6182caa8433..4c312d2ea7ed 100644
--- a/exporter/splunkhecexporter/client.go
+++ b/exporter/splunkhecexporter/client.go
@@ -161,20 +161,20 @@ func (c *client) pushLogData(ctx context.Context, ld pdata.Logs) (numDroppedLogs
 	c.wg.Add(1)
 	defer c.wg.Done()
 
-	gwriter := c.zippers.Get().(*gzip.Writer)
-	defer c.zippers.Put(gwriter)
+	gzipWriter := c.zippers.Get().(*gzip.Writer)
+	defer c.zippers.Put(gzipWriter)
 
 	gzipBuf := bytes.NewBuffer(make([]byte, 0, c.config.MaxContentLength))
-	gwriter.Reset(gzipBuf)
-	defer gwriter.Close()
+	gzipWriter.Reset(gzipBuf)
+	defer gzipWriter.Close()
 
-	ldWrapper := logDataWrapper{&ld}
-	eventsCh, cancel := ldWrapper.eventsInChunks(c.logger, c.config)
+	logs := logDataWrapper{&ld}
+	eventsCh, cancel := logs.eventsInChunks(c.logger, c.config)
 	defer cancel()
 
 	for events := range eventsCh {
 		if events.err != nil {
-			return ldWrapper.processErr(events.index, events.err)
+			return logs.numLogs(events.index), events.err
 		}
 
 		if events.buf.Len() == 0 {
@@ -184,23 +184,23 @@ func (c *client) pushLogData(ctx context.Context, ld pdata.Logs) (numDroppedLogs
 		// Not compressing if compression disabled or payload fit into a single ethernet frame.
 		if events.buf.Len() <= 1500 || c.config.DisableCompression {
 			if err = c.postEvents(ctx, events.buf, false); err != nil {
-				return ldWrapper.processErr(events.index, err)
+				return logs.numLogs(events.index), consumererror.PartialLogsError(err, *logs.subLogs(events.index))
 			}
 			continue
 		}
 
-		if _, err = gwriter.Write(events.buf.Bytes()); err != nil {
-			return ldWrapper.processErr(events.index, consumererror.Permanent(err))
+		if _, err = gzipWriter.Write(events.buf.Bytes()); err != nil {
+			return logs.numLogs(events.index), consumererror.Permanent(err)
 		}
 
-		gwriter.Flush()
+		gzipWriter.Flush()
 
 		if err = c.postEvents(ctx, gzipBuf, true); err != nil {
-			return ldWrapper.processErr(events.index, err)
+			return logs.numLogs(events.index), consumererror.PartialLogsError(err, *logs.subLogs(events.index))
 		}
 
 		gzipBuf.Reset()
-		gwriter.Reset(gzipBuf)
+		gzipWriter.Reset(gzipBuf)
 	}
 
 	return 0, nil
diff --git a/exporter/splunkhecexporter/logdata_to_splunk.go b/exporter/splunkhecexporter/logdata_to_splunk.go
index d5113d8963a1..344bf4c6f3a0 100644
--- a/exporter/splunkhecexporter/logdata_to_splunk.go
+++ b/exporter/splunkhecexporter/logdata_to_splunk.go
@@ -30,22 +30,22 @@ import (
 )
 
 // eventsBuf is a buffer of JSON encoded Splunk events.
-// The events are created from LogRecord(s) where one event maps to one LogRecord.
+// The events are created from LogRecord(s) where one event is created from one LogRecord.
 type eventsBuf struct {
 	buf *bytes.Buffer
-	// index is the eventIndex of the 1st event in buf.
-	index *eventIndex
+	// The logIndex of the LogRecord of 1st event in buf.
+	index *logIndex
 	err   error
 }
 
-// The index of an event composed of indices of the event's LogRecord.
-type eventIndex struct {
-	// Index of the LogRecord slice element from which the event is created.
-	log int
-	// Index of the InstrumentationLibraryLogs slice element parent of the LogRecord.
-	lib int
-	// Index of the ResourceLogs slice element parent of the InstrumentationLibraryLogs.
-	src int
+// Composite index of a log record.
+type logIndex struct {
+	// Index of a LogRecord slice element.
+	record int
+	// Index of the InstrumentationLibraryLogs slice element parent to the LogRecord.
+	library int
+	// Index of the ResourceLogs slice element parent to the InstrumentationLibraryLogs.
+	resource int
 }
 
 type logDataWrapper struct {
@@ -100,7 +100,7 @@ func (ld *logDataWrapper) eventsInChunks(logger *zap.Logger, config *Config) (ch
 
 							// Setting events index using the log record indices of the 1st event.
 							if events.index == nil {
-								events.index = &eventIndex{src: i, lib: j, log: k}
+								events.index = &logIndex{resource: i, library: j, record: k}
 							}
 
 							continue
@@ -112,7 +112,7 @@ func (ld *logDataWrapper) eventsInChunks(logger *zap.Logger, config *Config) (ch
 						events = &eventsBuf{buf: new(bytes.Buffer)}
 						// Setting events index using the log record indices of any current leftover event.
 						if event.Len() != 0 {
-							events.index = &eventIndex{src: i, lib: j, log: k}
+							events.index = &logIndex{resource: i, library: j, record: k}
 						}
 					}
 				}
@@ -128,43 +128,42 @@ func (ld *logDataWrapper) eventsInChunks(logger *zap.Logger, config *Config) (ch
 		eventsCh <- events
 
 	}()
+
 	return eventsCh, cancel
 }
 
-func (ld *logDataWrapper) countLogs(start *eventIndex) int {
+func (ld *logDataWrapper) numLogs(from *logIndex) int {
 	count, orig := 0, *ld.InternalRep().Orig
-	for i := start.src; i < len(orig); i++ {
-		for j, iLLogs := range orig[i].InstrumentationLibraryLogs {
+
+	// Validating logIndex. Invalid index will cause out of range panic.
+	_ = orig[from.resource].InstrumentationLibraryLogs[from.library].Logs[from.record]
+
+	for i := from.resource; i < len(orig); i++ {
+		for j, library := range orig[i].InstrumentationLibraryLogs {
 			switch {
-			case i == start.src && j < start.lib:
+			case i == from.resource && j < from.library:
 				continue
 			default:
-				count += len(iLLogs.Logs)
+				count += len(library.Logs)
 			}
 		}
 	}
-	return count - start.log
-}
 
-func (ld *logDataWrapper) trimLeft(end *eventIndex) *pdata.Logs {
-	clone := ld.Clone()
-	orig := *clone.InternalRep().Orig
-	orig = orig[end.src:]
-	orig[end.src].InstrumentationLibraryLogs = orig[end.src].InstrumentationLibraryLogs[end.lib:]
-	orig[end.src].InstrumentationLibraryLogs[end.lib].Logs = orig[end.src].InstrumentationLibraryLogs[end.lib].Logs[end.log:]
-	return &clone
+	return count - from.record
 }
 
-func (ld *logDataWrapper) processErr(index *eventIndex, err error) (int, error) {
-	if consumererror.IsPermanent(err) {
-		return ld.countLogs(index), err
-	}
+func (ld *logDataWrapper) subLogs(from *logIndex) *pdata.Logs {
+	clone := ld.Clone().InternalRep()
 
-	if _, ok := err.(consumererror.PartialError); ok {
-		failedLogs := ld.trimLeft(index)
-		return failedLogs.LogRecordCount(), consumererror.PartialLogsError(err, *failedLogs)
-	}
-	return ld.LogRecordCount(), err
+	subset := *clone.Orig
+	subset = subset[from.resource:]
+	subset[0].InstrumentationLibraryLogs = subset[0].InstrumentationLibraryLogs[from.library:]
+	subset[0].InstrumentationLibraryLogs[0].Logs = subset[0].InstrumentationLibraryLogs[0].Logs[from.record:]
+
+	clone.Orig = &subset
+	subsetLogs := pdata.LogsFromInternalRep(clone)
+
+	return &subsetLogs
 }
 
 func mapLogRecordToSplunkEvent(lr pdata.LogRecord, config *Config, logger *zap.Logger) *splunk.Event {
diff --git a/exporter/splunkhecexporter/logdata_to_splunk_test.go b/exporter/splunkhecexporter/logdata_to_splunk_test.go
index 14059fdc186f..4ac264a46684 100644
--- a/exporter/splunkhecexporter/logdata_to_splunk_test.go
+++ b/exporter/splunkhecexporter/logdata_to_splunk_test.go
@@ -139,29 +139,28 @@ func Test_logDataToSplunk(t *testing.T) {
 				commonLogSplunkEvent(float64(42), ts, map[string]interface{}{"custom": "custom"}, "myhost", "myapp", "myapp-type"),
 			},
 		},
-		// TODO: int64 body gets unmarshalled to float64 because splunk.Event.Event is of type interface{}
-		//{
-		//	name: "with int body",
-		//	logDataFn: func() pdata.Logs {
-		//		logRecord := pdata.NewLogRecord()
-		//		logRecord.Body().SetIntVal(42)
-		//		logRecord.Attributes().InsertString(conventions.AttributeServiceName, "myapp")
-		//		logRecord.Attributes().InsertString(splunk.SourcetypeLabel, "myapp-type")
-		//		logRecord.Attributes().InsertString(conventions.AttributeHostName, "myhost")
-		//		logRecord.Attributes().InsertString("custom", "custom")
-		//		logRecord.SetTimestamp(ts)
-		//		return makeLog(logRecord)
-		//	},
-		//	configDataFn: func() *Config {
-		//		return &Config{
-		//			Source:     "source",
-		//			SourceType: "sourcetype",
-		//		}
-		//	},
-		//	wantSplunkEvents: []*splunk.Event{
-		//		commonLogSplunkEvent(int64(42), ts, map[string]interface{}{"custom": "custom"}, "myhost", "myapp", "myapp-type"),
-		//	},
-		//},
+		{
+			name: "with int body",
+			logDataFn: func() pdata.Logs {
+				logRecord := pdata.NewLogRecord()
+				logRecord.Body().SetIntVal(42)
+				logRecord.Attributes().InsertString(conventions.AttributeServiceName, "myapp")
+				logRecord.Attributes().InsertString(splunk.SourcetypeLabel, "myapp-type")
+				logRecord.Attributes().InsertString(conventions.AttributeHostName, "myhost")
+				logRecord.Attributes().InsertString("custom", "custom")
+				logRecord.SetTimestamp(ts)
+				return makeLog(logRecord)
+			},
+			configDataFn: func() *Config {
+				return &Config{
+					Source:     "source",
+					SourceType: "sourcetype",
+				}
+			},
+			wantSplunkEvents: []*splunk.Event{
+				commonLogSplunkEvent(int64(42), ts, map[string]interface{}{"custom": "custom"}, "myhost", "myapp", "myapp-type"),
+			},
+		},
 		{
 			name: "with bool body",
 			logDataFn: func() pdata.Logs {
@@ -269,15 +268,22 @@ func Test_logDataToSplunk(t *testing.T) {
 
 			require.Equal(t, len(tt.wantSplunkEvents), len(events))
 
-			var gotEvent splunk.Event
-			var gotEvents []*splunk.Event
+			var got splunk.Event
+			var gots []*splunk.Event
 
 			for i, event := range events {
-				json.Unmarshal(event, &gotEvent)
-				assert.EqualValues(t, tt.wantSplunkEvents[i], &gotEvent)
-				gotEvents = append(gotEvents, &gotEvent)
+				json.Unmarshal(event, &got)
+				want := tt.wantSplunkEvents[i]
+				// float64 back to int64. int64 unmarshalled to float64 because Event is interface{}.
+				if _, ok := want.Event.(int64); ok {
+					if g, ok := got.Event.(float64); ok {
+						got.Event = int64(g)
+					}
+				}
+				assert.EqualValues(t, tt.wantSplunkEvents[i], &got)
+				gots = append(gots, &got)
 			}
-			assert.Equal(t, tt.wantSplunkEvents, gotEvents)
+			assert.Equal(t, tt.wantSplunkEvents, gots)
 		})
 	}
 }
@@ -311,8 +317,6 @@ func commonLogSplunkEvent(
 }
 
 func Test_nilLogs(t *testing.T) {
-	//events := logDataToSplunk(zap.NewNop(), pdata.NewLogs(), &Config{})
-	//assert.Equal(t, 0, len(events))
 	logs := pdata.NewLogs()
 	ldWrap := logDataWrapper{&logs}
 	eventsCh, cancel := ldWrap.eventsInChunks(zap.NewNop(), &Config{})
@@ -325,7 +329,6 @@ func Test_nilResourceLogs(t *testing.T) {
 	logs := pdata.NewLogs()
 	logs.ResourceLogs().Resize(1)
 	ldWrap := logDataWrapper{&logs}
-	//events := logDataToSplunk(zap.NewNop(), logs, &Config{})
 	eventsCh, cancel := ldWrap.eventsInChunks(zap.NewNop(), &Config{})
 	defer cancel()
 	events := <-eventsCh
@@ -333,12 +336,6 @@ func Test_nilResourceLogs(t *testing.T) {
 }
 
 func Test_nilInstrumentationLogs(t *testing.T) {
-	//logs := pdata.NewLogs()
-	//logs.ResourceLogs().Resize(1)
-	//resourceLog := logs.ResourceLogs().At(0)
-	//resourceLog.InstrumentationLibraryLogs().Resize(1)
-	//events := logDataToSplunk(zap.NewNop(), logs, &Config{})
-	//assert.Equal(t, 0, len(events))
 	logs := pdata.NewLogs()
 	logs.ResourceLogs().Resize(1)
 	resourceLog := logs.ResourceLogs().At(0)
@@ -358,3 +355,107 @@ func Test_nanoTimestampToEpochMilliseconds(t *testing.T) {
 	splunkTs = nanoTimestampToEpochMilliseconds(0)
 	assert.True(t, nil == splunkTs)
 }
+
+func Test_numLogs(t *testing.T) {
+	logs := logDataWrapper{&[]pdata.Logs{pdata.NewLogs()}[0]}
+	logs.ResourceLogs().Resize(2)
+
+	rl0 := logs.ResourceLogs().At(0)
+	rl0.InstrumentationLibraryLogs().Resize(2)
+	rl0.InstrumentationLibraryLogs().At(0).Logs().Append(pdata.NewLogRecord())
+	rl0.InstrumentationLibraryLogs().At(1).Logs().Append(pdata.NewLogRecord())
+	rl0.InstrumentationLibraryLogs().At(1).Logs().Append(pdata.NewLogRecord())
+
+	rl1 := logs.ResourceLogs().At(1)
+	rl1.InstrumentationLibraryLogs().Resize(3)
+	rl1.InstrumentationLibraryLogs().At(0).Logs().Append(pdata.NewLogRecord())
+	rl1.InstrumentationLibraryLogs().At(1).Logs().Append(pdata.NewLogRecord())
+	rl1.InstrumentationLibraryLogs().At(2).Logs().Append(pdata.NewLogRecord())
+
+	// Indices of LogRecord(s) created.
+	//     0            1      <- ResourceLogs parent index
+	//    / \         / | \
+	//   0   1      0  1  2    <- InstrumentationLibraryLogs parent index
+	//  /   / \    /  /  /
+	// 0   0   1  0  0  0      <- LogRecord index
+
+	_0_0_0 := &logIndex{resource: 0, library: 0, record: 0}
+	got := logs.numLogs(_0_0_0)
+	assert.Equal(t, 6, got)
+
+	_0_1_1 := &logIndex{resource: 0, library: 1, record: 1}
+	got = logs.numLogs(_0_1_1)
+	assert.Equal(t, 4, got)
+}
+
+func Test_subLogs(t *testing.T) {
+	logs := logDataWrapper{&[]pdata.Logs{pdata.NewLogs()}[0]}
+	logs.ResourceLogs().Resize(2)
+
+	rl0 := logs.ResourceLogs().At(0)
+	rl0.InstrumentationLibraryLogs().Resize(2)
+
+	log := pdata.NewLogRecord()
+	log.SetName("(0, 0, 0)")
+	rl0.InstrumentationLibraryLogs().At(0).Logs().Append(log)
+
+	log = pdata.NewLogRecord()
+	log.SetName("(0, 1, 0)")
+	rl0.InstrumentationLibraryLogs().At(1).Logs().Append(log)
+
+	log = pdata.NewLogRecord()
+	log.SetName("(0, 1, 1)")
+	rl0.InstrumentationLibraryLogs().At(1).Logs().Append(log)
+
+	rl1 := logs.ResourceLogs().At(1)
+	rl1.InstrumentationLibraryLogs().Resize(3)
+
+	log = pdata.NewLogRecord()
+	log.SetName("(1, 0, 0)")
+	rl1.InstrumentationLibraryLogs().At(0).Logs().Append(log)
+
+	log = pdata.NewLogRecord()
+	log.SetName("(1, 1, 0)")
+	rl1.InstrumentationLibraryLogs().At(1).Logs().Append(log)
+
+	log = pdata.NewLogRecord()
+	log.SetName("(1, 2, 0)")
+	rl1.InstrumentationLibraryLogs().At(2).Logs().Append(log)
+
+	// Indices of LogRecord(s) created.
+	//     0            1      <- ResourceLogs parent index
+	//    / \         / | \
+	//   0   1      0  1  2    <- InstrumentationLibraryLogs parent index
+	//  /   / \    /  /  /
+	// 0   0   1  0  0  0      <- LogRecord index
+
+	// Logs subset from leftmost index.
+	_0_0_0 := &logIndex{resource: 0, library: 0, record: 0}
+	got := logDataWrapper{logs.subLogs(_0_0_0)}
+
+	assert.Equal(t, 6, got.numLogs(_0_0_0))
+	orig := *got.InternalRep().Orig
+	assert.Equal(t, "(0, 0, 0)", orig[0].InstrumentationLibraryLogs[0].Logs[0].Name)
+	assert.Equal(t, "(0, 1, 0)", orig[0].InstrumentationLibraryLogs[1].Logs[0].Name)
+	assert.Equal(t, "(0, 1, 1)", orig[0].InstrumentationLibraryLogs[1].Logs[1].Name)
+	assert.Equal(t, "(1, 0, 0)", orig[1].InstrumentationLibraryLogs[0].Logs[0].Name)
+	assert.Equal(t, "(1, 1, 0)", orig[1].InstrumentationLibraryLogs[1].Logs[0].Name)
+	assert.Equal(t, "(1, 2, 0)", orig[1].InstrumentationLibraryLogs[2].Logs[0].Name)
+
+	// Logs subset from rightmost index.
+	_1_2_0 := &logIndex{resource: 1, library: 2, record: 0}
+	got = logDataWrapper{logs.subLogs(_1_2_0)}
+
+	assert.Equal(t, 1, got.numLogs(_0_0_0))
+	orig = *got.InternalRep().Orig
+	assert.Equal(t, "(1, 2, 0)", orig[0].InstrumentationLibraryLogs[0].Logs[0].Name)
+
+	// Logs subset from an in-between index.
+	_1_1_0 := &logIndex{resource: 1, library: 1, record: 0}
+	got = logDataWrapper{logs.subLogs(_1_1_0)}
+
+	assert.Equal(t, 2, got.numLogs(_0_0_0))
+	orig = *got.InternalRep().Orig
+	assert.Equal(t, "(1, 1, 0)", orig[0].InstrumentationLibraryLogs[0].Logs[0].Name)
+	assert.Equal(t, "(1, 2, 0)", orig[0].InstrumentationLibraryLogs[1].Logs[0].Name)
+}

From cec865c874acdceda7c91c1babde3849229a3e37 Mon Sep 17 00:00:00 2001
From: Bremer Jonathan <bjonathan@splunk.com>
Date: Thu, 4 Mar 2021 00:59:48 -0500
Subject: [PATCH 03/26] Add unit tests

---
 exporter/splunkhecexporter/client.go          |  22 +-
 .../splunkhecexporter/logdata_to_splunk.go    |  85 ++--
 .../logdata_to_splunk_test.go                 | 362 ++++++++++++++----
 3 files changed, 348 insertions(+), 121 deletions(-)

diff --git a/exporter/splunkhecexporter/client.go b/exporter/splunkhecexporter/client.go
index 4c312d2ea7ed..80af2d204032 100644
--- a/exporter/splunkhecexporter/client.go
+++ b/exporter/splunkhecexporter/client.go
@@ -169,34 +169,34 @@ func (c *client) pushLogData(ctx context.Context, ld pdata.Logs) (numDroppedLogs
 	defer gzipWriter.Close()
 
 	logs := logDataWrapper{&ld}
-	eventsCh, cancel := logs.eventsInChunks(c.logger, c.config)
+	chunkCh, cancel := logs.chunkEvents(c.logger, c.config)
 	defer cancel()
 
-	for events := range eventsCh {
-		if events.err != nil {
-			return logs.numLogs(events.index), events.err
+	for chunk := range chunkCh {
+		if chunk.err != nil {
+			return logs.numLogs(chunk.index), chunk.err
 		}
 
-		if events.buf.Len() == 0 {
+		if chunk.buf.Len() == 0 {
 			continue
 		}
 
 		// Not compressing if compression disabled or payload fit into a single ethernet frame.
-		if events.buf.Len() <= 1500 || c.config.DisableCompression {
-			if err = c.postEvents(ctx, events.buf, false); err != nil {
-				return logs.numLogs(events.index), consumererror.PartialLogsError(err, *logs.subLogs(events.index))
+		if chunk.buf.Len() <= 1500 || c.config.DisableCompression {
+			if err = c.postEvents(ctx, chunk.buf, false); err != nil {
+				return logs.numLogs(chunk.index), consumererror.PartialLogsError(err, *logs.subLogs(chunk.index))
 			}
 			continue
 		}
 
-		if _, err = gzipWriter.Write(events.buf.Bytes()); err != nil {
-			return logs.numLogs(events.index), consumererror.Permanent(err)
+		if _, err = gzipWriter.Write(chunk.buf.Bytes()); err != nil {
+			return logs.numLogs(chunk.index), consumererror.Permanent(err)
 		}
 
 		gzipWriter.Flush()
 
 		if err = c.postEvents(ctx, gzipBuf, true); err != nil {
-			return logs.numLogs(events.index), consumererror.PartialLogsError(err, *logs.subLogs(events.index))
+			return logs.numLogs(chunk.index), consumererror.PartialLogsError(err, *logs.subLogs(chunk.index))
 		}
 
 		gzipBuf.Reset()
diff --git a/exporter/splunkhecexporter/logdata_to_splunk.go b/exporter/splunkhecexporter/logdata_to_splunk.go
index 344bf4c6f3a0..a2fdf133724f 100644
--- a/exporter/splunkhecexporter/logdata_to_splunk.go
+++ b/exporter/splunkhecexporter/logdata_to_splunk.go
@@ -29,42 +29,42 @@ import (
 	"github.com/open-telemetry/opentelemetry-collector-contrib/internal/splunk"
 )
 
-// eventsBuf is a buffer of JSON encoded Splunk events.
+// eventsChunk buffers JSON encoded Splunk events.
 // The events are created from LogRecord(s) where one event is created from one LogRecord.
-type eventsBuf struct {
+type eventsChunk struct {
 	buf *bytes.Buffer
 	// The logIndex of the LogRecord of 1st event in buf.
 	index *logIndex
 	err   error
 }
 
-// Composite index of a log record.
+// Composite index of a log record in pdata.Logs.
 type logIndex struct {
-	// Index of a LogRecord slice element.
-	record int
-	// Index of the InstrumentationLibraryLogs slice element parent to the LogRecord.
-	library int
-	// Index of the ResourceLogs slice element parent to the InstrumentationLibraryLogs.
-	resource int
+	// Index in orig list (i.e. root parent index).
+	origIdx int
+	// Index in InstrumentationLibraryLogs list (i.e. immediate parent index).
+	instIdx int
+	// Index in Logs list (i.e. the log record index).
+	logsIdx int
 }
 
 type logDataWrapper struct {
 	*pdata.Logs
 }
 
-func (ld *logDataWrapper) eventsInChunks(logger *zap.Logger, config *Config) (chan *eventsBuf, context.CancelFunc) {
+func (ld *logDataWrapper) chunkEvents(logger *zap.Logger, config *Config) (chan *eventsChunk, context.CancelFunc) {
 	ctx, cancel := context.WithCancel(context.Background())
-	eventsCh := make(chan *eventsBuf)
+	chunkCh := make(chan *eventsChunk)
 
 	go func() {
-		defer close(eventsCh)
+		defer close(chunkCh)
 
 		// event buffers a single event.
 		event := new(bytes.Buffer)
 		encoder := json.NewEncoder(event)
 
-		// events buffers events up to the max content length.
-		events := &eventsBuf{buf: new(bytes.Buffer)}
+		// chunk buffers events up to the max content length.
+		chunk := &eventsChunk{buf: new(bytes.Buffer)}
 
 		rl := ld.ResourceLogs()
 		for i := 0; i < rl.Len(); i++ {
@@ -77,71 +77,68 @@ func (ld *logDataWrapper) eventsInChunks(logger *zap.Logger, config *Config) (ch
 						return
 					default:
 						if err := encoder.Encode(mapLogRecordToSplunkEvent(l.At(k), config, logger)); err != nil {
-							eventsCh <- &eventsBuf{buf: nil, index: nil, err: consumererror.Permanent(err)}
+							chunkCh <- &eventsChunk{buf: nil, index: nil, err: consumererror.Permanent(err)}
 							return
 						}
 						event.WriteString("\r\n\r\n")
 
+					addToChunk:
 						// The size of an event must be less than or equal to max content length.
 						if config.MaxContentLength > 0 && event.Len() > config.MaxContentLength {
-							err := fmt.Errorf("found a log event bigger than max content length (event: %d bytes, max: %d bytes)", config.MaxContentLength, event.Len())
-							eventsCh <- &eventsBuf{buf: nil, index: nil, err: consumererror.Permanent(err)}
+							chunkCh <- &eventsChunk{
+								buf:   nil,
+								index: nil,
+								err:   consumererror.Permanent(fmt.Errorf("log event bytes exceed max content length configured (log: %d, max: %d)", event.Len(), config.MaxContentLength)),
+							}
 							return
 						}
 
-						// Moving the event to events.buf if length will be <= max content length.
+						// Moving the event to chunk.buf if length will be <= max content length.
 						// Max content length <= 0 is interpreted as unbound.
-						if events.buf.Len()+event.Len() <= config.MaxContentLength || config.MaxContentLength <= 0 {
+						if chunk.buf.Len()+event.Len() <= config.MaxContentLength || config.MaxContentLength <= 0 {
 							// WriteTo() empties and resets buffer event.
-							if _, err := event.WriteTo(events.buf); err != nil {
-								eventsCh <- &eventsBuf{buf: nil, index: nil, err: consumererror.Permanent(err)}
+							if _, err := event.WriteTo(chunk.buf); err != nil {
+								chunkCh <- &eventsChunk{buf: nil, index: nil, err: consumererror.Permanent(err)}
 								return
 							}
 
-							// Setting events index using the log record indices of the 1st event.
-							if events.index == nil {
-								events.index = &logIndex{resource: i, library: j, record: k}
+							// Setting chunk index using the log logsIdx indices of the 1st event.
+							if chunk.index == nil {
+								chunk.index = &logIndex{origIdx: i, instIdx: j, logsIdx: k}
 							}
 
 							continue
 						}
 
-						eventsCh <- events
+						chunkCh <- chunk
 
 						// Creating a new events buffer.
-						events = &eventsBuf{buf: new(bytes.Buffer)}
-						// Setting events index using the log record indices of any current leftover event.
+						chunk = &eventsChunk{buf: new(bytes.Buffer)}
+						// Setting chunk index using the log logsIdx indices of any current leftover event.
 						if event.Len() != 0 {
-							events.index = &logIndex{resource: i, library: j, record: k}
+							goto addToChunk
 						}
 					}
 				}
 			}
 		}
 
-		// Writing any leftover event to eventsBuf buffer `events.buf`.
-		if _, err := event.WriteTo(events.buf); err != nil {
-			eventsCh <- &eventsBuf{buf: nil, index: nil, err: consumererror.Permanent(err)}
-			return
-		}
-
-		eventsCh <- events
-
+		chunkCh <- chunk
 	}()
 
-	return eventsCh, cancel
+	return chunkCh, cancel
 }
 
 func (ld *logDataWrapper) numLogs(from *logIndex) int {
 	count, orig := 0, *ld.InternalRep().Orig
 
 	// Validating logIndex. Invalid index will cause out of range panic.
-	_ = orig[from.resource].InstrumentationLibraryLogs[from.library].Logs[from.record]
+	_ = orig[from.origIdx].InstrumentationLibraryLogs[from.instIdx].Logs[from.logsIdx]
 
-	for i := from.resource; i < len(orig); i++ {
+	for i := from.origIdx; i < len(orig); i++ {
 		for j, library := range orig[i].InstrumentationLibraryLogs {
 			switch {
-			case i == from.resource && j < from.library:
+			case i == from.origIdx && j < from.instIdx:
 				continue
 			default:
 				count += len(library.Logs)
@@ -149,16 +146,16 @@ func (ld *logDataWrapper) numLogs(from *logIndex) int {
 		}
 	}
 
-	return count - from.record
+	return count - from.logsIdx
 }
 
 func (ld *logDataWrapper) subLogs(from *logIndex) *pdata.Logs {
 	clone := ld.Clone().InternalRep()
 
 	subset := *clone.Orig
-	subset = subset[from.resource:]
-	subset[0].InstrumentationLibraryLogs = subset[0].InstrumentationLibraryLogs[from.library:]
-	subset[0].InstrumentationLibraryLogs[0].Logs = subset[0].InstrumentationLibraryLogs[0].Logs[from.record:]
+	subset = subset[from.origIdx:]
+	subset[0].InstrumentationLibraryLogs = subset[0].InstrumentationLibraryLogs[from.instIdx:]
+	subset[0].InstrumentationLibraryLogs[0].Logs = subset[0].InstrumentationLibraryLogs[0].Logs[from.logsIdx:]
 
 	clone.Orig = &subset
 	subsetLogs := pdata.LogsFromInternalRep(clone)
diff --git a/exporter/splunkhecexporter/logdata_to_splunk_test.go b/exporter/splunkhecexporter/logdata_to_splunk_test.go
index 4ac264a46684..d19eae44bffb 100644
--- a/exporter/splunkhecexporter/logdata_to_splunk_test.go
+++ b/exporter/splunkhecexporter/logdata_to_splunk_test.go
@@ -17,6 +17,7 @@ package splunkhecexporter
 import (
 	"bytes"
 	"encoding/json"
+	"math"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -28,7 +29,7 @@ import (
 	"github.com/open-telemetry/opentelemetry-collector-contrib/internal/splunk"
 )
 
-func Test_logDataToSplunk(t *testing.T) {
+func Test_chunkEvents(t *testing.T) {
 	logger := zap.NewNop()
 	ts := pdata.TimestampUnixNano(123)
 
@@ -256,15 +257,15 @@ func Test_logDataToSplunk(t *testing.T) {
 			},
 		},
 	}
+
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			logs := tt.logDataFn()
-			logsWrapper := logDataWrapper{&logs}
+			logs := logDataWrapper{&[]pdata.Logs{tt.logDataFn()}[0]}
 
-			ch, cancel := logsWrapper.eventsInChunks(logger, tt.configDataFn())
+			eventsCh, cancel := logs.chunkEvents(logger, tt.configDataFn())
 			defer cancel()
 
-			events := bytes.Split(bytes.TrimSpace((<-ch).buf.Bytes()), []byte("\r\n\r\n"))
+			events := bytes.Split(bytes.TrimSpace((<-eventsCh).buf.Bytes()), []byte("\r\n\r\n"))
 
 			require.Equal(t, len(tt.wantSplunkEvents), len(events))
 
@@ -288,6 +289,198 @@ func Test_logDataToSplunk(t *testing.T) {
 	}
 }
 
+func Test_chunkEvents_MaxContentLength_AllEventsInChunk(t *testing.T) {
+	logs := testLogs()
+
+	_, max, events := jsonEncodeEventsBytes(logs, &Config{})
+
+	eventsLength := 0
+	for _, event := range events {
+		eventsLength += len(event)
+	}
+
+	// Chunk max content length to fit all events in 1 chunk.
+	chunkLength := len(events) * max
+	config := Config{MaxContentLength: chunkLength}
+
+	chunkCh, cancel := logs.chunkEvents(zap.NewNop(), &config)
+	defer cancel()
+
+	numChunks := 0
+
+	for chunk := range chunkCh {
+		assert.Nil(t, chunk.err)
+		assert.Len(t, chunk.buf.Bytes(), eventsLength)
+		numChunks++
+	}
+
+	assert.Equal(t, 1, numChunks)
+}
+
+func Test_chunkEvents_MaxContentLength_0(t *testing.T) {
+	logs := testLogs()
+
+	_, _, events := jsonEncodeEventsBytes(logs, &Config{})
+
+	eventsLength := 0
+	for _, event := range events {
+		eventsLength += len(event)
+	}
+
+	// Chunk max content length 0 is interpreted as unlimited length.
+	chunkLength := 0
+	config := Config{MaxContentLength: chunkLength}
+
+	chunkCh, cancel := logs.chunkEvents(zap.NewNop(), &config)
+	defer cancel()
+
+	numChunks := 0
+
+	for chunk := range chunkCh {
+		assert.Nil(t, chunk.err)
+		assert.Len(t, chunk.buf.Bytes(), eventsLength)
+		numChunks++
+	}
+
+	assert.Equal(t, 1, numChunks)
+}
+
+func Test_chunkEvents_MaxContentLength_Negative(t *testing.T) {
+	logs := testLogs()
+
+	_, _, events := jsonEncodeEventsBytes(logs, &Config{})
+
+	eventsLength := 0
+	for _, event := range events {
+		eventsLength += len(event)
+	}
+
+	// Negative max content length is interpreted as unlimited length.
+	chunkLength := -3
+	config := Config{MaxContentLength: chunkLength}
+
+	chunkCh, cancel := logs.chunkEvents(zap.NewNop(), &config)
+	defer cancel()
+
+	numChunks := 0
+
+	for chunk := range chunkCh {
+		assert.Nil(t, chunk.err)
+		assert.Len(t, chunk.buf.Bytes(), eventsLength)
+		numChunks++
+	}
+
+	assert.Equal(t, 1, numChunks)
+}
+
+func Test_chunkEvents_MaxContentLength_Small(t *testing.T) {
+	logs := testLogs()
+
+	min, _, _ := jsonEncodeEventsBytes(logs, &Config{})
+
+	// Chunk max content length less than all events.
+	chunkLength := min - 1
+	config := Config{MaxContentLength: chunkLength}
+
+	chunkCh, cancel := logs.chunkEvents(zap.NewNop(), &config)
+	defer cancel()
+
+	numChunks := 0
+
+	for chunk := range chunkCh {
+		if chunk.err == nil {
+			numChunks++
+		}
+		assert.Nil(t, chunk.buf)
+		assert.Nil(t, chunk.index)
+		assert.Contains(t, chunk.err.Error(), "log event bytes exceed max content length configured")
+	}
+
+	assert.Equal(t, 0, numChunks)
+}
+
+func Test_chunkEvents_MaxContentLength_1EventPerChunk(t *testing.T) {
+	logs := testLogs()
+
+	min, max, events := jsonEncodeEventsBytes(logs, &Config{})
+
+	numEvents := len(events)
+
+	// Chunk max content length = max and this condition results in 1 event per chunk.
+	assert.True(t, min >= max/2)
+
+	chunkLength := max
+	config := Config{MaxContentLength: chunkLength}
+
+	chunkCh, cancel := logs.chunkEvents(zap.NewNop(), &config)
+	defer cancel()
+
+	numChunks := 0
+
+	for chunk := range chunkCh {
+		assert.Nil(t, chunk.err)
+		assert.Len(t, chunk.buf.Bytes(), len(events[numChunks]))
+		numChunks++
+	}
+
+	// 1 event per chunk.
+	assert.Equal(t, numEvents, numChunks)
+}
+
+func Test_chunkEvents_MaxContentLength_2EventsPerChunk(t *testing.T) {
+	logs := testLogs()
+
+	min, max, events := jsonEncodeEventsBytes(logs, &Config{})
+
+	numEvents := len(events)
+
+	// Chunk max content length = 2 * max and this condition results in 2 event per chunk.
+	assert.True(t, min >= max/2)
+
+	chunkLength := 2 * max
+	config := Config{MaxContentLength: chunkLength}
+
+	// Config max content length equal to the length of the largest event.
+	chunkCh, cancel := logs.chunkEvents(zap.NewNop(), &config)
+	defer cancel()
+
+	numChunks := 0
+
+	for chunk := range chunkCh {
+		assert.Nil(t, chunk.err)
+		numChunks++
+	}
+
+	// 2 events per chunk.
+	assert.Equal(t, numEvents, 2*numChunks)
+}
+
+func Test_chunkEvents_JSONEncodeError(t *testing.T) {
+	logs := testLogs()
+
+	// Setting a log logsIdx body to +Inf
+	logs.ResourceLogs().At(0).
+		InstrumentationLibraryLogs().At(0).
+		Logs().At(0).
+		Body().SetDoubleVal(math.Inf(1))
+
+	// JSON Encoding +Inf should trigger unsupported value error
+	config := &Config{}
+	eventsCh, cancel := logs.chunkEvents(zap.NewNop(), config)
+	defer cancel()
+
+	// event should contain an unsupported value error triggered by JSON Encoding +Inf
+	event := <-eventsCh
+
+	assert.Nil(t, event.buf)
+	assert.Nil(t, event.index)
+	assert.Contains(t, event.err.Error(), "json: unsupported value: +Inf")
+
+	// the error should cause the channel to be closed.
+	_, ok := <-eventsCh
+	assert.True(t, !ok, "Events channel should be closed on error")
+}
+
 func makeLog(record pdata.LogRecord) pdata.Logs {
 	logs := pdata.NewLogs()
 	logs.ResourceLogs().Resize(1)
@@ -319,7 +512,7 @@ func commonLogSplunkEvent(
 func Test_nilLogs(t *testing.T) {
 	logs := pdata.NewLogs()
 	ldWrap := logDataWrapper{&logs}
-	eventsCh, cancel := ldWrap.eventsInChunks(zap.NewNop(), &Config{})
+	eventsCh, cancel := ldWrap.chunkEvents(zap.NewNop(), &Config{})
 	defer cancel()
 	events := <-eventsCh
 	assert.Equal(t, 0, events.buf.Len())
@@ -329,7 +522,7 @@ func Test_nilResourceLogs(t *testing.T) {
 	logs := pdata.NewLogs()
 	logs.ResourceLogs().Resize(1)
 	ldWrap := logDataWrapper{&logs}
-	eventsCh, cancel := ldWrap.eventsInChunks(zap.NewNop(), &Config{})
+	eventsCh, cancel := ldWrap.chunkEvents(zap.NewNop(), &Config{})
 	defer cancel()
 	events := <-eventsCh
 	assert.Equal(t, 0, events.buf.Len())
@@ -341,7 +534,7 @@ func Test_nilInstrumentationLogs(t *testing.T) {
 	resourceLog := logs.ResourceLogs().At(0)
 	resourceLog.InstrumentationLibraryLogs().Resize(1)
 	ldWrap := logDataWrapper{&logs}
-	eventsCh, cancel := ldWrap.eventsInChunks(zap.NewNop(), &Config{})
+	eventsCh, cancel := ldWrap.chunkEvents(zap.NewNop(), &Config{})
 	defer cancel()
 	events := <-eventsCh
 	assert.Equal(t, 0, events.buf.Len())
@@ -357,38 +550,77 @@ func Test_nanoTimestampToEpochMilliseconds(t *testing.T) {
 }
 
 func Test_numLogs(t *testing.T) {
-	logs := logDataWrapper{&[]pdata.Logs{pdata.NewLogs()}[0]}
-	logs.ResourceLogs().Resize(2)
+	// See nested structure of logs in testLogs() comments.
+	logs := testLogs()
 
-	rl0 := logs.ResourceLogs().At(0)
-	rl0.InstrumentationLibraryLogs().Resize(2)
-	rl0.InstrumentationLibraryLogs().At(0).Logs().Append(pdata.NewLogRecord())
-	rl0.InstrumentationLibraryLogs().At(1).Logs().Append(pdata.NewLogRecord())
-	rl0.InstrumentationLibraryLogs().At(1).Logs().Append(pdata.NewLogRecord())
-
-	rl1 := logs.ResourceLogs().At(1)
-	rl1.InstrumentationLibraryLogs().Resize(3)
-	rl1.InstrumentationLibraryLogs().At(0).Logs().Append(pdata.NewLogRecord())
-	rl1.InstrumentationLibraryLogs().At(1).Logs().Append(pdata.NewLogRecord())
-	rl1.InstrumentationLibraryLogs().At(2).Logs().Append(pdata.NewLogRecord())
-
-	// Indices of LogRecord(s) created.
-	//     0            1      <- ResourceLogs parent index
-	//    / \         / | \
-	//   0   1      0  1  2    <- InstrumentationLibraryLogs parent index
-	//  /   / \    /  /  /
-	// 0   0   1  0  0  0      <- LogRecord index
-
-	_0_0_0 := &logIndex{resource: 0, library: 0, record: 0}
+	_0_0_0 := &logIndex{origIdx: 0, instIdx: 0, logsIdx: 0}
 	got := logs.numLogs(_0_0_0)
+
 	assert.Equal(t, 6, got)
 
-	_0_1_1 := &logIndex{resource: 0, library: 1, record: 1}
+	_0_1_1 := &logIndex{origIdx: 0, instIdx: 1, logsIdx: 1}
 	got = logs.numLogs(_0_1_1)
+
 	assert.Equal(t, 4, got)
 }
 
 func Test_subLogs(t *testing.T) {
+	// See nested structure of logs in testLogs() comments.
+	logs := testLogs()
+
+	// Logs subset from leftmost index.
+	_0_0_0 := &logIndex{origIdx: 0, instIdx: 0, logsIdx: 0}
+	got := logDataWrapper{logs.subLogs(_0_0_0)}
+
+	assert.Equal(t, 6, got.numLogs(_0_0_0))
+
+	orig := *got.InternalRep().Orig
+
+	assert.Equal(t, "(0, 0, 0)", orig[0].InstrumentationLibraryLogs[0].Logs[0].Name)
+	assert.Equal(t, "(0, 1, 0)", orig[0].InstrumentationLibraryLogs[1].Logs[0].Name)
+	assert.Equal(t, "(0, 1, 1)", orig[0].InstrumentationLibraryLogs[1].Logs[1].Name)
+	assert.Equal(t, "(1, 0, 0)", orig[1].InstrumentationLibraryLogs[0].Logs[0].Name)
+	assert.Equal(t, "(1, 1, 0)", orig[1].InstrumentationLibraryLogs[1].Logs[0].Name)
+	assert.Equal(t, "(1, 2, 0)", orig[1].InstrumentationLibraryLogs[2].Logs[0].Name)
+
+	// Logs subset from rightmost index.
+	_1_2_0 := &logIndex{origIdx: 1, instIdx: 2, logsIdx: 0}
+	got = logDataWrapper{logs.subLogs(_1_2_0)}
+
+	assert.Equal(t, 1, got.numLogs(_0_0_0))
+
+	orig = *got.InternalRep().Orig
+
+	assert.Equal(t, "(1, 2, 0)", orig[0].InstrumentationLibraryLogs[0].Logs[0].Name)
+
+	// Logs subset from an in-between index.
+	_1_1_0 := &logIndex{origIdx: 1, instIdx: 1, logsIdx: 0}
+	got = logDataWrapper{logs.subLogs(_1_1_0)}
+
+	assert.Equal(t, 2, got.numLogs(_0_0_0))
+
+	orig = *got.InternalRep().Orig
+
+	assert.Equal(t, "(1, 1, 0)", orig[0].InstrumentationLibraryLogs[0].Logs[0].Name)
+	assert.Equal(t, "(1, 2, 0)", orig[0].InstrumentationLibraryLogs[1].Logs[0].Name)
+}
+
+// Creates pdata.Logs for testing.
+//
+// Structure of the pdata.Logs created showing indices:
+//
+//     0            1      <- orig index
+//    / \         / | \
+//   0   1      0  1  2    <- InstrumentationLibraryLogs index
+//  /   / \    /  /  /
+// 0   0   1  0  0  0      <- Logs index
+//
+// The log records are named in the pattern:
+// (<orig index>, <InstrumentationLibraryLogs index>, <Logs index>)
+//
+// The log records are about the same size and some test depend this fact.
+//
+func testLogs() *logDataWrapper {
 	logs := logDataWrapper{&[]pdata.Logs{pdata.NewLogs()}[0]}
 	logs.ResourceLogs().Resize(2)
 
@@ -422,40 +654,38 @@ func Test_subLogs(t *testing.T) {
 	log.SetName("(1, 2, 0)")
 	rl1.InstrumentationLibraryLogs().At(2).Logs().Append(log)
 
-	// Indices of LogRecord(s) created.
-	//     0            1      <- ResourceLogs parent index
-	//    / \         / | \
-	//   0   1      0  1  2    <- InstrumentationLibraryLogs parent index
-	//  /   / \    /  /  /
-	// 0   0   1  0  0  0      <- LogRecord index
-
-	// Logs subset from leftmost index.
-	_0_0_0 := &logIndex{resource: 0, library: 0, record: 0}
-	got := logDataWrapper{logs.subLogs(_0_0_0)}
-
-	assert.Equal(t, 6, got.numLogs(_0_0_0))
-	orig := *got.InternalRep().Orig
-	assert.Equal(t, "(0, 0, 0)", orig[0].InstrumentationLibraryLogs[0].Logs[0].Name)
-	assert.Equal(t, "(0, 1, 0)", orig[0].InstrumentationLibraryLogs[1].Logs[0].Name)
-	assert.Equal(t, "(0, 1, 1)", orig[0].InstrumentationLibraryLogs[1].Logs[1].Name)
-	assert.Equal(t, "(1, 0, 0)", orig[1].InstrumentationLibraryLogs[0].Logs[0].Name)
-	assert.Equal(t, "(1, 1, 0)", orig[1].InstrumentationLibraryLogs[1].Logs[0].Name)
-	assert.Equal(t, "(1, 2, 0)", orig[1].InstrumentationLibraryLogs[2].Logs[0].Name)
-
-	// Logs subset from rightmost index.
-	_1_2_0 := &logIndex{resource: 1, library: 2, record: 0}
-	got = logDataWrapper{logs.subLogs(_1_2_0)}
-
-	assert.Equal(t, 1, got.numLogs(_0_0_0))
-	orig = *got.InternalRep().Orig
-	assert.Equal(t, "(1, 2, 0)", orig[0].InstrumentationLibraryLogs[0].Logs[0].Name)
-
-	// Logs subset from an in-between index.
-	_1_1_0 := &logIndex{resource: 1, library: 1, record: 0}
-	got = logDataWrapper{logs.subLogs(_1_1_0)}
+	return &logs
+}
 
-	assert.Equal(t, 2, got.numLogs(_0_0_0))
-	orig = *got.InternalRep().Orig
-	assert.Equal(t, "(1, 1, 0)", orig[0].InstrumentationLibraryLogs[0].Logs[0].Name)
-	assert.Equal(t, "(1, 2, 0)", orig[0].InstrumentationLibraryLogs[1].Logs[0].Name)
+func jsonEncodeEventsBytes(logs *logDataWrapper, config *Config) (int, int, [][]byte) {
+	events := make([][]byte, 0)
+	// min, max number of bytes of smallest, largest events.
+	var min, max int
+
+	event := new(bytes.Buffer)
+	encoder := json.NewEncoder(event)
+
+	rl := logs.ResourceLogs()
+	for i := 0; i < rl.Len(); i++ {
+		ill := rl.At(i).InstrumentationLibraryLogs()
+		for j := 0; j < ill.Len(); j++ {
+			l := ill.At(j).Logs()
+			for k := 0; k < l.Len(); k++ {
+				if err := encoder.Encode(mapLogRecordToSplunkEvent(l.At(k), config, zap.NewNop())); err == nil {
+					event.WriteString("\r\n\r\n")
+					dst := make([]byte, len(event.Bytes()))
+					copy(dst, event.Bytes())
+					events = append(events, dst)
+					if event.Len() < min || min == 0 {
+						min = event.Len()
+					}
+					if event.Len() > max {
+						max = event.Len()
+					}
+					event.Reset()
+				}
+			}
+		}
+	}
+	return min, max, events
 }

From 0fe0b4d26cbe49a69dd1785d83bfba12b21e8619 Mon Sep 17 00:00:00 2001
From: Bremer Jonathan <bjonathan@splunk.com>
Date: Thu, 4 Mar 2021 09:36:14 -0500
Subject: [PATCH 04/26] Update comment

---
 exporter/splunkhecexporter/logdata_to_splunk.go | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/exporter/splunkhecexporter/logdata_to_splunk.go b/exporter/splunkhecexporter/logdata_to_splunk.go
index a2fdf133724f..e41b55f07e42 100644
--- a/exporter/splunkhecexporter/logdata_to_splunk.go
+++ b/exporter/splunkhecexporter/logdata_to_splunk.go
@@ -114,7 +114,8 @@ func (ld *logDataWrapper) chunkEvents(logger *zap.Logger, config *Config) (chan
 
 						// Creating a new events buffer.
 						chunk = &eventsChunk{buf: new(bytes.Buffer)}
-						// Setting chunk index using the log logsIdx indices of any current leftover event.
+
+						// Adding remaining event to the new chunk
 						if event.Len() != 0 {
 							goto addToChunk
 						}

From 4333f912a573989480a91750b47e8530d99f0aff Mon Sep 17 00:00:00 2001
From: Bremer Jonathan <bjonathan@splunk.com>
Date: Thu, 4 Mar 2021 12:17:07 -0500
Subject: [PATCH 05/26] Add test case to cover context cancellation

---
 exporter/splunkhecexporter/client.go          |   5 +-
 .../splunkhecexporter/logdata_to_splunk.go    |  92 +++++++------
 .../logdata_to_splunk_test.go                 | 121 +++++++++++++-----
 3 files changed, 141 insertions(+), 77 deletions(-)

diff --git a/exporter/splunkhecexporter/client.go b/exporter/splunkhecexporter/client.go
index 80af2d204032..332ae1b5a2a2 100644
--- a/exporter/splunkhecexporter/client.go
+++ b/exporter/splunkhecexporter/client.go
@@ -169,9 +169,12 @@ func (c *client) pushLogData(ctx context.Context, ld pdata.Logs) (numDroppedLogs
 	defer gzipWriter.Close()
 
 	logs := logDataWrapper{&ld}
-	chunkCh, cancel := logs.chunkEvents(c.logger, c.config)
+
+	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
+	chunkCh := logs.chunkEvents(ctx, c.logger, c.config)
+
 	for chunk := range chunkCh {
 		if chunk.err != nil {
 			return logs.numLogs(chunk.index), chunk.err
diff --git a/exporter/splunkhecexporter/logdata_to_splunk.go b/exporter/splunkhecexporter/logdata_to_splunk.go
index e41b55f07e42..3b98a67cdaa6 100644
--- a/exporter/splunkhecexporter/logdata_to_splunk.go
+++ b/exporter/splunkhecexporter/logdata_to_splunk.go
@@ -52,8 +52,7 @@ type logDataWrapper struct {
 	*pdata.Logs
 }
 
-func (ld *logDataWrapper) chunkEvents(logger *zap.Logger, config *Config) (chan *eventsChunk, context.CancelFunc) {
-	ctx, cancel := context.WithCancel(context.Background())
+func (ld *logDataWrapper) chunkEvents(ctx context.Context, logger *zap.Logger, config *Config) chan *eventsChunk {
 	chunkCh := make(chan *eventsChunk)
 
 	go func() {
@@ -72,62 +71,73 @@ func (ld *logDataWrapper) chunkEvents(logger *zap.Logger, config *Config) (chan
 			for j := 0; j < ill.Len(); j++ {
 				l := ill.At(j).Logs()
 				for k := 0; k < l.Len(); k++ {
-					select {
-					case <-ctx.Done():
+					if err := encoder.Encode(mapLogRecordToSplunkEvent(l.At(k), config, logger)); err != nil {
+						select {
+						case <-ctx.Done():
+						case chunkCh <- &eventsChunk{buf: nil, index: nil, err: consumererror.Permanent(err)}:
+						}
 						return
-					default:
-						if err := encoder.Encode(mapLogRecordToSplunkEvent(l.At(k), config, logger)); err != nil {
-							chunkCh <- &eventsChunk{buf: nil, index: nil, err: consumererror.Permanent(err)}
-							return
+					}
+					event.WriteString("\r\n\r\n")
+
+				addToChunk:
+					// The size of an event must be less than or equal to max content length.
+					if config.MaxContentLength > 0 && event.Len() > config.MaxContentLength {
+						select {
+						case <-ctx.Done():
+						case chunkCh <- &eventsChunk{
+							buf:   nil,
+							index: nil,
+							err:   consumererror.Permanent(fmt.Errorf("log event bytes exceed max content length configured (log: %d, max: %d)", event.Len(), config.MaxContentLength))}:
 						}
-						event.WriteString("\r\n\r\n")
-
-					addToChunk:
-						// The size of an event must be less than or equal to max content length.
-						if config.MaxContentLength > 0 && event.Len() > config.MaxContentLength {
-							chunkCh <- &eventsChunk{
-								buf:   nil,
-								index: nil,
-								err:   consumererror.Permanent(fmt.Errorf("log event bytes exceed max content length configured (log: %d, max: %d)", event.Len(), config.MaxContentLength)),
+						return
+					}
+
+					// Moving the event to chunk.buf if length will be <= max content length.
+					// Max content length <= 0 is interpreted as unbound.
+					if chunk.buf.Len()+event.Len() <= config.MaxContentLength || config.MaxContentLength <= 0 {
+						// WriteTo() empties and resets buffer event.
+						if _, err := event.WriteTo(chunk.buf); err != nil {
+							select {
+							case <-ctx.Done():
+							case chunkCh <- &eventsChunk{buf: nil, index: nil, err: consumererror.Permanent(err)}:
 							}
 							return
 						}
 
-						// Moving the event to chunk.buf if length will be <= max content length.
-						// Max content length <= 0 is interpreted as unbound.
-						if chunk.buf.Len()+event.Len() <= config.MaxContentLength || config.MaxContentLength <= 0 {
-							// WriteTo() empties and resets buffer event.
-							if _, err := event.WriteTo(chunk.buf); err != nil {
-								chunkCh <- &eventsChunk{buf: nil, index: nil, err: consumererror.Permanent(err)}
-								return
-							}
-
-							// Setting chunk index using the log logsIdx indices of the 1st event.
-							if chunk.index == nil {
-								chunk.index = &logIndex{origIdx: i, instIdx: j, logsIdx: k}
-							}
-
-							continue
+						// Setting chunk index using the log logsIdx indices of the 1st event.
+						if chunk.index == nil {
+							chunk.index = &logIndex{origIdx: i, instIdx: j, logsIdx: k}
 						}
 
-						chunkCh <- chunk
+						continue
+					}
+
+					select {
+					case <-ctx.Done():
+						return
+					case chunkCh <- chunk:
+					}
 
-						// Creating a new events buffer.
-						chunk = &eventsChunk{buf: new(bytes.Buffer)}
+					// Creating a new events buffer.
+					chunk = &eventsChunk{buf: new(bytes.Buffer)}
 
-						// Adding remaining event to the new chunk
-						if event.Len() != 0 {
-							goto addToChunk
-						}
+					// Adding remaining event to the new chunk
+					if event.Len() != 0 {
+						goto addToChunk
 					}
 				}
 			}
 		}
 
-		chunkCh <- chunk
+		select {
+		case <-ctx.Done():
+			return
+		case chunkCh <- chunk:
+		}
 	}()
 
-	return chunkCh, cancel
+	return chunkCh
 }
 
 func (ld *logDataWrapper) numLogs(from *logIndex) int {
diff --git a/exporter/splunkhecexporter/logdata_to_splunk_test.go b/exporter/splunkhecexporter/logdata_to_splunk_test.go
index d19eae44bffb..b15d18befa75 100644
--- a/exporter/splunkhecexporter/logdata_to_splunk_test.go
+++ b/exporter/splunkhecexporter/logdata_to_splunk_test.go
@@ -16,6 +16,7 @@ package splunkhecexporter
 
 import (
 	"bytes"
+	"context"
 	"encoding/json"
 	"math"
 	"testing"
@@ -262,10 +263,12 @@ func Test_chunkEvents(t *testing.T) {
 		t.Run(tt.name, func(t *testing.T) {
 			logs := logDataWrapper{&[]pdata.Logs{tt.logDataFn()}[0]}
 
-			eventsCh, cancel := logs.chunkEvents(logger, tt.configDataFn())
+			ctx, cancel := context.WithCancel(context.Background())
 			defer cancel()
 
-			events := bytes.Split(bytes.TrimSpace((<-eventsCh).buf.Bytes()), []byte("\r\n\r\n"))
+			chunkCh := logs.chunkEvents(ctx, logger, tt.configDataFn())
+
+			events := bytes.Split(bytes.TrimSpace((<-chunkCh).buf.Bytes()), []byte("\r\n\r\n"))
 
 			require.Equal(t, len(tt.wantSplunkEvents), len(events))
 
@@ -303,9 +306,11 @@ func Test_chunkEvents_MaxContentLength_AllEventsInChunk(t *testing.T) {
 	chunkLength := len(events) * max
 	config := Config{MaxContentLength: chunkLength}
 
-	chunkCh, cancel := logs.chunkEvents(zap.NewNop(), &config)
+	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
+	chunkCh := logs.chunkEvents(ctx, zap.NewNop(), &config)
+
 	numChunks := 0
 
 	for chunk := range chunkCh {
@@ -331,9 +336,11 @@ func Test_chunkEvents_MaxContentLength_0(t *testing.T) {
 	chunkLength := 0
 	config := Config{MaxContentLength: chunkLength}
 
-	chunkCh, cancel := logs.chunkEvents(zap.NewNop(), &config)
+	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
+	chunkCh := logs.chunkEvents(ctx, zap.NewNop(), &config)
+
 	numChunks := 0
 
 	for chunk := range chunkCh {
@@ -359,9 +366,11 @@ func Test_chunkEvents_MaxContentLength_Negative(t *testing.T) {
 	chunkLength := -3
 	config := Config{MaxContentLength: chunkLength}
 
-	chunkCh, cancel := logs.chunkEvents(zap.NewNop(), &config)
+	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
+	chunkCh := logs.chunkEvents(ctx, zap.NewNop(), &config)
+
 	numChunks := 0
 
 	for chunk := range chunkCh {
@@ -382,9 +391,11 @@ func Test_chunkEvents_MaxContentLength_Small(t *testing.T) {
 	chunkLength := min - 1
 	config := Config{MaxContentLength: chunkLength}
 
-	chunkCh, cancel := logs.chunkEvents(zap.NewNop(), &config)
+	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
+	chunkCh := logs.chunkEvents(ctx, zap.NewNop(), &config)
+
 	numChunks := 0
 
 	for chunk := range chunkCh {
@@ -402,19 +413,23 @@ func Test_chunkEvents_MaxContentLength_Small(t *testing.T) {
 func Test_chunkEvents_MaxContentLength_1EventPerChunk(t *testing.T) {
 	logs := testLogs()
 
-	min, max, events := jsonEncodeEventsBytes(logs, &Config{})
+	minLength, maxLength, events := jsonEncodeEventsBytes(logs, &Config{})
 
 	numEvents := len(events)
 
-	// Chunk max content length = max and this condition results in 1 event per chunk.
-	assert.True(t, min >= max/2)
+	assert.True(t, numEvents > 1, "More than 1 event required")
 
-	chunkLength := max
+	assert.True(t, minLength >= maxLength/2, "Smallest event >= half largest event required")
+
+	// Setting chunk length to have 1 event per chunk.
+	chunkLength := maxLength
 	config := Config{MaxContentLength: chunkLength}
 
-	chunkCh, cancel := logs.chunkEvents(zap.NewNop(), &config)
+	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 
+	chunkCh := logs.chunkEvents(ctx, zap.NewNop(), &config)
+
 	numChunks := 0
 
 	for chunk := range chunkCh {
@@ -423,10 +438,42 @@ func Test_chunkEvents_MaxContentLength_1EventPerChunk(t *testing.T) {
 		numChunks++
 	}
 
-	// 1 event per chunk.
+	// Number of chunks should equal number of events.
 	assert.Equal(t, numEvents, numChunks)
 }
 
+func Test_chunkEvents_Cancel(t *testing.T) {
+	logs := testLogs()
+
+	min, max, events := jsonEncodeEventsBytes(logs, &Config{})
+
+	assert.True(t, len(events) > 1, "More than 1 event required")
+
+	assert.True(t, min >= max/2, "Smallest event >= half largest event required")
+
+	// Setting chunk length to have as many chunks as there are events.
+	chunkLength := max
+	config := Config{MaxContentLength: chunkLength}
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	chunkCh := logs.chunkEvents(ctx, zap.NewNop(), &config)
+
+	_, ok := <-chunkCh
+
+	assert.True(t, ok, "Chunk channel open before cancel")
+
+	cancel()
+
+	// chan chuckCh maybe selected over cancel's Done chan when read immediately.
+	// Reading chuckCh more than once guarantees the selection of Done chan.
+	<-chunkCh
+	_, ok = <-chunkCh
+
+	assert.True(t, !ok, "Chunk channel closed after cancel")
+
+}
+
 func Test_chunkEvents_MaxContentLength_2EventsPerChunk(t *testing.T) {
 	logs := testLogs()
 
@@ -440,8 +487,11 @@ func Test_chunkEvents_MaxContentLength_2EventsPerChunk(t *testing.T) {
 	chunkLength := 2 * max
 	config := Config{MaxContentLength: chunkLength}
 
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
 	// Config max content length equal to the length of the largest event.
-	chunkCh, cancel := logs.chunkEvents(zap.NewNop(), &config)
+	chunkCh := logs.chunkEvents(ctx, zap.NewNop(), &config)
 	defer cancel()
 
 	numChunks := 0
@@ -466,18 +516,19 @@ func Test_chunkEvents_JSONEncodeError(t *testing.T) {
 
 	// JSON Encoding +Inf should trigger unsupported value error
 	config := &Config{}
-	eventsCh, cancel := logs.chunkEvents(zap.NewNop(), config)
+	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
+	chunkCh := logs.chunkEvents(ctx, zap.NewNop(), config)
 
-	// event should contain an unsupported value error triggered by JSON Encoding +Inf
-	event := <-eventsCh
+	// chunk should contain an unsupported value error triggered by JSON Encoding +Inf
+	chunk := <-chunkCh
 
-	assert.Nil(t, event.buf)
-	assert.Nil(t, event.index)
-	assert.Contains(t, event.err.Error(), "json: unsupported value: +Inf")
+	assert.Nil(t, chunk.buf)
+	assert.Nil(t, chunk.index)
+	assert.Contains(t, chunk.err.Error(), "json: unsupported value: +Inf")
 
 	// the error should cause the channel to be closed.
-	_, ok := <-eventsCh
+	_, ok := <-chunkCh
 	assert.True(t, !ok, "Events channel should be closed on error")
 }
 
@@ -510,34 +561,34 @@ func commonLogSplunkEvent(
 }
 
 func Test_nilLogs(t *testing.T) {
-	logs := pdata.NewLogs()
-	ldWrap := logDataWrapper{&logs}
-	eventsCh, cancel := ldWrap.chunkEvents(zap.NewNop(), &Config{})
+	logs := logDataWrapper{&[]pdata.Logs{pdata.NewLogs()}[0]}
+	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	events := <-eventsCh
-	assert.Equal(t, 0, events.buf.Len())
+	chunkCh := logs.chunkEvents(ctx, zap.NewNop(), &Config{})
+	chunk := <-chunkCh
+	assert.Equal(t, 0, chunk.buf.Len())
 }
 
 func Test_nilResourceLogs(t *testing.T) {
-	logs := pdata.NewLogs()
+	logs := logDataWrapper{&[]pdata.Logs{pdata.NewLogs()}[0]}
 	logs.ResourceLogs().Resize(1)
-	ldWrap := logDataWrapper{&logs}
-	eventsCh, cancel := ldWrap.chunkEvents(zap.NewNop(), &Config{})
+	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	events := <-eventsCh
-	assert.Equal(t, 0, events.buf.Len())
+	chunkCh := logs.chunkEvents(ctx, zap.NewNop(), &Config{})
+	chunk := <-chunkCh
+	assert.Equal(t, 0, chunk.buf.Len())
 }
 
 func Test_nilInstrumentationLogs(t *testing.T) {
-	logs := pdata.NewLogs()
+	logs := logDataWrapper{&[]pdata.Logs{pdata.NewLogs()}[0]}
 	logs.ResourceLogs().Resize(1)
 	resourceLog := logs.ResourceLogs().At(0)
 	resourceLog.InstrumentationLibraryLogs().Resize(1)
-	ldWrap := logDataWrapper{&logs}
-	eventsCh, cancel := ldWrap.chunkEvents(zap.NewNop(), &Config{})
+	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	events := <-eventsCh
-	assert.Equal(t, 0, events.buf.Len())
+	chunkCh := logs.chunkEvents(ctx, zap.NewNop(), &Config{})
+	chunk := <-chunkCh
+	assert.Equal(t, 0, chunk.buf.Len())
 }
 
 func Test_nanoTimestampToEpochMilliseconds(t *testing.T) {

From 592ea05196cca04cb09acd985c583f14aaa3a190 Mon Sep 17 00:00:00 2001
From: Bremer Jonathan <bjonathan@splunk.com>
Date: Thu, 4 Mar 2021 14:53:30 -0500
Subject: [PATCH 06/26] Add test covering pushing invalid logs

---
 exporter/splunkhecexporter/client_test.go     | 44 +++++++++++++++++++
 .../splunkhecexporter/logdata_to_splunk.go    | 17 ++++---
 2 files changed, 52 insertions(+), 9 deletions(-)

diff --git a/exporter/splunkhecexporter/client_test.go b/exporter/splunkhecexporter/client_test.go
index 993701b6b4b1..08c6a4199b0b 100644
--- a/exporter/splunkhecexporter/client_test.go
+++ b/exporter/splunkhecexporter/client_test.go
@@ -26,6 +26,8 @@ import (
 	"testing"
 	"time"
 
+	"go.opentelemetry.io/collector/consumer/consumererror"
+
 	"github.com/stretchr/testify/assert"
 	"go.opentelemetry.io/collector/component"
 	"go.opentelemetry.io/collector/component/componenttest"
@@ -447,3 +449,45 @@ func TestInvalidURLClient(t *testing.T) {
 	err := c.sendSplunkEvents(context.Background(), []*splunk.Event{})
 	assert.EqualError(t, err, "Permanent error: parse \"//in%20va%20lid\": invalid URL escape \"%20\"")
 }
+
+func Test_pushLogData_InvalidLog(t *testing.T) {
+	c := client{
+		zippers: sync.Pool{New: func() interface{} {
+			return gzip.NewWriter(nil)
+		}},
+		config: &Config{},
+	}
+
+	logs := pdata.NewLogs()
+	logs.ResourceLogs().Resize(1)
+	logs.ResourceLogs().At(0).InstrumentationLibraryLogs().Resize(1)
+
+	log := pdata.NewLogRecord()
+	// Invalid log value
+	log.Body().SetDoubleVal(math.Inf(1))
+	logs.ResourceLogs().At(0).InstrumentationLibraryLogs().At(0).Logs().Append(log)
+
+	numDroppedLogs, err := c.pushLogData(context.Background(), logs)
+
+	assert.Contains(t, err.Error(), "json: unsupported value: +Inf")
+	assert.Equal(t, 1, numDroppedLogs)
+}
+
+func Test_pushLogData_PostError(t *testing.T) {
+	c := client{
+		url: &url.URL{Host: "in va lid"},
+		zippers: sync.Pool{New: func() interface{} {
+			return gzip.NewWriter(nil)
+		}},
+		config: &Config{DisableCompression: false},
+	}
+
+	numLogs := 1500
+	logs := createLogData(numLogs)
+
+	numDroppedLogs, err := c.pushLogData(context.Background(), logs)
+	if assert.Error(t, err) {
+		assert.IsType(t, consumererror.PartialError{}, err)
+		assert.Equal(t, numLogs, numDroppedLogs)
+	}
+}
diff --git a/exporter/splunkhecexporter/logdata_to_splunk.go b/exporter/splunkhecexporter/logdata_to_splunk.go
index 3b98a67cdaa6..c02a6eebd6aa 100644
--- a/exporter/splunkhecexporter/logdata_to_splunk.go
+++ b/exporter/splunkhecexporter/logdata_to_splunk.go
@@ -71,10 +71,14 @@ func (ld *logDataWrapper) chunkEvents(ctx context.Context, logger *zap.Logger, c
 			for j := 0; j < ill.Len(); j++ {
 				l := ill.At(j).Logs()
 				for k := 0; k < l.Len(); k++ {
+					if chunk.index == nil {
+						chunk.index = &logIndex{origIdx: i, instIdx: j, logsIdx: k}
+					}
+
 					if err := encoder.Encode(mapLogRecordToSplunkEvent(l.At(k), config, logger)); err != nil {
 						select {
 						case <-ctx.Done():
-						case chunkCh <- &eventsChunk{buf: nil, index: nil, err: consumererror.Permanent(err)}:
+						case chunkCh <- &eventsChunk{buf: nil, index: chunk.index, err: consumererror.Permanent(err)}:
 						}
 						return
 					}
@@ -87,7 +91,7 @@ func (ld *logDataWrapper) chunkEvents(ctx context.Context, logger *zap.Logger, c
 						case <-ctx.Done():
 						case chunkCh <- &eventsChunk{
 							buf:   nil,
-							index: nil,
+							index: chunk.index,
 							err:   consumererror.Permanent(fmt.Errorf("log event bytes exceed max content length configured (log: %d, max: %d)", event.Len(), config.MaxContentLength))}:
 						}
 						return
@@ -100,16 +104,10 @@ func (ld *logDataWrapper) chunkEvents(ctx context.Context, logger *zap.Logger, c
 						if _, err := event.WriteTo(chunk.buf); err != nil {
 							select {
 							case <-ctx.Done():
-							case chunkCh <- &eventsChunk{buf: nil, index: nil, err: consumererror.Permanent(err)}:
+							case chunkCh <- &eventsChunk{buf: nil, index: chunk.index, err: consumererror.Permanent(err)}:
 							}
 							return
 						}
-
-						// Setting chunk index using the log logsIdx indices of the 1st event.
-						if chunk.index == nil {
-							chunk.index = &logIndex{origIdx: i, instIdx: j, logsIdx: k}
-						}
-
 						continue
 					}
 
@@ -124,6 +122,7 @@ func (ld *logDataWrapper) chunkEvents(ctx context.Context, logger *zap.Logger, c
 
 					// Adding remaining event to the new chunk
 					if event.Len() != 0 {
+						chunk.index = &logIndex{origIdx: i, instIdx: j, logsIdx: k}
 						goto addToChunk
 					}
 				}

From 7006372031e1e8f2ac6c09d608af22c798eb9a01 Mon Sep 17 00:00:00 2001
From: Bremer Jonathan <bjonathan@splunk.com>
Date: Thu, 4 Mar 2021 23:21:32 -0500
Subject: [PATCH 07/26] Add unit tests

---
 .../logdata_to_splunk_test.go                 | 36 ++++++++++++++++---
 1 file changed, 31 insertions(+), 5 deletions(-)

diff --git a/exporter/splunkhecexporter/logdata_to_splunk_test.go b/exporter/splunkhecexporter/logdata_to_splunk_test.go
index b15d18befa75..9bcd91c86aae 100644
--- a/exporter/splunkhecexporter/logdata_to_splunk_test.go
+++ b/exporter/splunkhecexporter/logdata_to_splunk_test.go
@@ -20,6 +20,7 @@ import (
 	"encoding/json"
 	"math"
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -465,9 +466,9 @@ func Test_chunkEvents_Cancel(t *testing.T) {
 
 	cancel()
 
-	// chan chuckCh maybe selected over cancel's Done chan when read immediately.
-	// Reading chuckCh more than once guarantees the selection of Done chan.
-	<-chunkCh
+	// Giving time for chunkCh to close.
+	time.Sleep(time.Millisecond)
+
 	_, ok = <-chunkCh
 
 	assert.True(t, !ok, "Chunk channel closed after cancel")
@@ -515,10 +516,10 @@ func Test_chunkEvents_JSONEncodeError(t *testing.T) {
 		Body().SetDoubleVal(math.Inf(1))
 
 	// JSON Encoding +Inf should trigger unsupported value error
-	config := &Config{}
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
-	chunkCh := logs.chunkEvents(ctx, zap.NewNop(), config)
+
+	chunkCh := logs.chunkEvents(ctx, zap.NewNop(), &Config{})
 
 	// chunk should contain an unsupported value error triggered by JSON Encoding +Inf
 	chunk := <-chunkCh
@@ -532,6 +533,31 @@ func Test_chunkEvents_JSONEncodeError(t *testing.T) {
 	assert.True(t, !ok, "Events channel should be closed on error")
 }
 
+func Test_chunkEvents_JSONEncodeError_Cancel(t *testing.T) {
+	logs := testLogs()
+
+	// Setting a log logsIdx body to +Inf
+	logs.ResourceLogs().At(0).
+		InstrumentationLibraryLogs().At(0).
+		Logs().At(0).
+		Body().SetDoubleVal(math.Inf(1))
+
+	// JSON Encoding +Inf should trigger unsupported value error
+	ctx, cancel := context.WithCancel(context.Background())
+
+	chunkCh := logs.chunkEvents(ctx, zap.NewNop(), &Config{})
+
+	cancel()
+
+	// Giving time for chunkCh to close.
+	time.Sleep(time.Millisecond)
+
+	chunk, ok := <-chunkCh
+
+	assert.Nil(t, chunk)
+	assert.True(t, !ok, "Cancel should close events channel")
+}
+
 func makeLog(record pdata.LogRecord) pdata.Logs {
 	logs := pdata.NewLogs()
 	logs.ResourceLogs().Resize(1)

From 0b6169b8d2534278e0a50bb473d4607f3f31f28f Mon Sep 17 00:00:00 2001
From: Bremer Jonathan <bjonathan@splunk.com>
Date: Thu, 4 Mar 2021 23:31:29 -0500
Subject: [PATCH 08/26] Fix failing tests

---
 exporter/splunkhecexporter/logdata_to_splunk_test.go | 2 --
 1 file changed, 2 deletions(-)

diff --git a/exporter/splunkhecexporter/logdata_to_splunk_test.go b/exporter/splunkhecexporter/logdata_to_splunk_test.go
index 9bcd91c86aae..648d73370a36 100644
--- a/exporter/splunkhecexporter/logdata_to_splunk_test.go
+++ b/exporter/splunkhecexporter/logdata_to_splunk_test.go
@@ -404,7 +404,6 @@ func Test_chunkEvents_MaxContentLength_Small(t *testing.T) {
 			numChunks++
 		}
 		assert.Nil(t, chunk.buf)
-		assert.Nil(t, chunk.index)
 		assert.Contains(t, chunk.err.Error(), "log event bytes exceed max content length configured")
 	}
 
@@ -525,7 +524,6 @@ func Test_chunkEvents_JSONEncodeError(t *testing.T) {
 	chunk := <-chunkCh
 
 	assert.Nil(t, chunk.buf)
-	assert.Nil(t, chunk.index)
 	assert.Contains(t, chunk.err.Error(), "json: unsupported value: +Inf")
 
 	// the error should cause the channel to be closed.

From 8fce7e2c4d3b6560207246dbff6d31a5b05f0f50 Mon Sep 17 00:00:00 2001
From: Bremer Jonathan <bjonathan@splunk.com>
Date: Thu, 4 Mar 2021 23:49:03 -0500
Subject: [PATCH 09/26] Fix impi verification error

---
 exporter/splunkhecexporter/client_test.go | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/exporter/splunkhecexporter/client_test.go b/exporter/splunkhecexporter/client_test.go
index 08c6a4199b0b..cc7c3d161582 100644
--- a/exporter/splunkhecexporter/client_test.go
+++ b/exporter/splunkhecexporter/client_test.go
@@ -26,11 +26,10 @@ import (
 	"testing"
 	"time"
 
-	"go.opentelemetry.io/collector/consumer/consumererror"
-
 	"github.com/stretchr/testify/assert"
 	"go.opentelemetry.io/collector/component"
 	"go.opentelemetry.io/collector/component/componenttest"
+	"go.opentelemetry.io/collector/consumer/consumererror"
 	"go.opentelemetry.io/collector/consumer/pdata"
 	"go.opentelemetry.io/collector/translator/conventions"
 	"go.uber.org/zap"

From 076d46782f39c9793f7d2e72fc5b8537a66c376e Mon Sep 17 00:00:00 2001
From: Bremer Jonathan <bjonathan@splunk.com>
Date: Mon, 8 Mar 2021 12:04:33 -0500
Subject: [PATCH 10/26] Add unit tests

---
 .../logdata_to_splunk_test.go                 | 71 ++++++++++++++++++-
 1 file changed, 68 insertions(+), 3 deletions(-)

diff --git a/exporter/splunkhecexporter/logdata_to_splunk_test.go b/exporter/splunkhecexporter/logdata_to_splunk_test.go
index 648d73370a36..5d6649da83e1 100644
--- a/exporter/splunkhecexporter/logdata_to_splunk_test.go
+++ b/exporter/splunkhecexporter/logdata_to_splunk_test.go
@@ -353,6 +353,40 @@ func Test_chunkEvents_MaxContentLength_0(t *testing.T) {
 	assert.Equal(t, 1, numChunks)
 }
 
+func Test_chunkEvents_MaxContentLength_0_Cancel(t *testing.T) {
+	logs := testLogs()
+
+	_, _, events := jsonEncodeEventsBytes(logs, &Config{})
+
+	eventsLength := 0
+	for _, event := range events {
+		eventsLength += len(event)
+	}
+
+	// Chunk max content length 0 is interpreted as unlimited length.
+	chunkLength := 0
+	config := Config{MaxContentLength: chunkLength}
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	chunkCh := logs.chunkEvents(ctx, zap.NewNop(), &config)
+
+	cancel()
+
+	// Giving time for chunkCh to close.
+	time.Sleep(time.Millisecond)
+
+	numChunks := 0
+
+	for chunk := range chunkCh {
+		assert.Nil(t, chunk.buf)
+		assert.Nil(t, chunk.err)
+		numChunks++
+	}
+
+	assert.Equal(t, 0, numChunks)
+}
+
 func Test_chunkEvents_MaxContentLength_Negative(t *testing.T) {
 	logs := testLogs()
 
@@ -383,12 +417,12 @@ func Test_chunkEvents_MaxContentLength_Negative(t *testing.T) {
 	assert.Equal(t, 1, numChunks)
 }
 
-func Test_chunkEvents_MaxContentLength_Small(t *testing.T) {
+func Test_chunkEvents_MaxContentLength_Small_Error(t *testing.T) {
 	logs := testLogs()
 
 	min, _, _ := jsonEncodeEventsBytes(logs, &Config{})
 
-	// Chunk max content length less than all events.
+	// Configuring max content length to be smaller than event lengths.
 	chunkLength := min - 1
 	config := Config{MaxContentLength: chunkLength}
 
@@ -410,6 +444,37 @@ func Test_chunkEvents_MaxContentLength_Small(t *testing.T) {
 	assert.Equal(t, 0, numChunks)
 }
 
+func Test_chunkEvents_MaxContentLength_Small_Error_Cancel(t *testing.T) {
+	logs := testLogs()
+
+	min, _, _ := jsonEncodeEventsBytes(logs, &Config{})
+
+	// Configuring max content length to be smaller than event lengths.
+	chunkLength := min - 1
+	config := Config{MaxContentLength: chunkLength}
+
+	ctx, cancel := context.WithCancel(context.Background())
+
+	chunkCh := logs.chunkEvents(ctx, zap.NewNop(), &config)
+
+	cancel()
+
+	// Giving time for chunkCh to close.
+	time.Sleep(time.Millisecond)
+
+	numChunks := 0
+
+	for chunk := range chunkCh {
+		if chunk.err == nil {
+			numChunks++
+		}
+		assert.Nil(t, chunk.buf)
+		assert.Nil(t, chunk.err)
+	}
+
+	assert.Equal(t, 0, numChunks)
+}
+
 func Test_chunkEvents_MaxContentLength_1EventPerChunk(t *testing.T) {
 	logs := testLogs()
 
@@ -442,7 +507,7 @@ func Test_chunkEvents_MaxContentLength_1EventPerChunk(t *testing.T) {
 	assert.Equal(t, numEvents, numChunks)
 }
 
-func Test_chunkEvents_Cancel(t *testing.T) {
+func Test_chunkEvents_MaxContentLength_1EventPerChunk_Cancel(t *testing.T) {
 	logs := testLogs()
 
 	min, max, events := jsonEncodeEventsBytes(logs, &Config{})

From 4d3f4f4fd566f2146ed8a95a8b16b7fdc77e9947 Mon Sep 17 00:00:00 2001
From: Bremer Jonathan <bjonathan@splunk.com>
Date: Tue, 9 Mar 2021 17:42:46 -0500
Subject: [PATCH 11/26] Update docs

---
 exporter/splunkhecexporter/README.md          |  1 +
 exporter/splunkhecexporter/client.go          |  2 +-
 exporter/splunkhecexporter/config.go          |  4 +-
 exporter/splunkhecexporter/config_test.go     | 14 +++---
 exporter/splunkhecexporter/factory.go         | 18 ++++----
 .../splunkhecexporter/logdata_to_splunk.go    |  6 +--
 .../logdata_to_splunk_test.go                 | 44 +++++++++----------
 7 files changed, 45 insertions(+), 44 deletions(-)

diff --git a/exporter/splunkhecexporter/README.md b/exporter/splunkhecexporter/README.md
index 9b32c813a59e..3638736e4197 100644
--- a/exporter/splunkhecexporter/README.md
+++ b/exporter/splunkhecexporter/README.md
@@ -22,6 +22,7 @@ The following configuration options can also be configured:
 - `disable_compression` (default: false): Whether to disable gzip compression over HTTP.
 - `timeout` (default: 10s): HTTP timeout when sending data.
 - `insecure_skip_verify` (default: false): Whether to skip checking the certificate of the HEC endpoint when sending data over HTTPS.
+- `max_content_length_logs` (default: 1048576): Maximum log data size in bytes per HTTP post limited to 1048576 bytes (1Mib).
 
 In addition, this exporter offers queued retry which is enabled by default.
 Information about queued retry configuration parameters can be found
diff --git a/exporter/splunkhecexporter/client.go b/exporter/splunkhecexporter/client.go
index 332ae1b5a2a2..fee982040491 100644
--- a/exporter/splunkhecexporter/client.go
+++ b/exporter/splunkhecexporter/client.go
@@ -164,7 +164,7 @@ func (c *client) pushLogData(ctx context.Context, ld pdata.Logs) (numDroppedLogs
 	gzipWriter := c.zippers.Get().(*gzip.Writer)
 	defer c.zippers.Put(gzipWriter)
 
-	gzipBuf := bytes.NewBuffer(make([]byte, 0, c.config.MaxContentLength))
+	gzipBuf := bytes.NewBuffer(make([]byte, 0, c.config.MaxContentLengthLogs))
 	gzipWriter.Reset(gzipBuf)
 	defer gzipWriter.Close()
 
diff --git a/exporter/splunkhecexporter/config.go b/exporter/splunkhecexporter/config.go
index 1da8e1f3bf9e..dc8dc89cfcf3 100644
--- a/exporter/splunkhecexporter/config.go
+++ b/exporter/splunkhecexporter/config.go
@@ -61,8 +61,8 @@ type Config struct {
 	// insecure_skip_verify skips checking the certificate of the HEC endpoint when sending data over HTTPS. Defaults to false.
 	InsecureSkipVerify bool `mapstructure:"insecure_skip_verify"`
 
-	// MaxContentLength is the Splunk HEC endpoint content length limit. Defaults to 1Mib, the current limit.
-	MaxContentLength int `mapstructure:"max_content_length"`
+	// MaxContentLengthLogs is the maximum log data size per HTTP post limited to 1048576 bytes (1Mib). Defaults to 1048576.
+	MaxContentLengthLogs int `mapstructure:"max_content_length_logs"`
 }
 
 func (cfg *Config) getOptionsFromConfig() (*exporterOptions, error) {
diff --git a/exporter/splunkhecexporter/config_test.go b/exporter/splunkhecexporter/config_test.go
index 29f17f0292e2..e2e81c470d9f 100644
--- a/exporter/splunkhecexporter/config_test.go
+++ b/exporter/splunkhecexporter/config_test.go
@@ -58,13 +58,13 @@ func TestLoadConfig(t *testing.T) {
 			TypeVal: configmodels.Type(typeStr),
 			NameVal: expectedName,
 		},
-		Token:            "00000000-0000-0000-0000-0000000000000",
-		Endpoint:         "https://splunk:8088/services/collector",
-		Source:           "otel",
-		SourceType:       "otel",
-		Index:            "metrics",
-		MaxConnections:   100,
-		MaxContentLength: 1024 * 1024,
+		Token:                "00000000-0000-0000-0000-0000000000000",
+		Endpoint:             "https://splunk:8088/services/collector",
+		Source:               "otel",
+		SourceType:           "otel",
+		Index:                "metrics",
+		MaxConnections:       100,
+		MaxContentLengthLogs: 1024 * 1024,
 		TimeoutSettings: exporterhelper.TimeoutSettings{
 			Timeout: 10 * time.Second,
 		},
diff --git a/exporter/splunkhecexporter/factory.go b/exporter/splunkhecexporter/factory.go
index 6b1c1f29ee9c..bb828274039f 100644
--- a/exporter/splunkhecexporter/factory.go
+++ b/exporter/splunkhecexporter/factory.go
@@ -26,10 +26,10 @@ import (
 
 const (
 	// The value of "type" key in configuration.
-	typeStr                 = "splunk_hec"
-	defaultMaxIdleCons      = 100
-	defaultHTTPTimeout      = 10 * time.Second
-	defaultMaxContentLength = 1024 * 1024
+	typeStr                     = "splunk_hec"
+	defaultMaxIdleCons          = 100
+	defaultHTTPTimeout          = 10 * time.Second
+	defaultMaxContentLengthLogs = 1024 * 1024
 )
 
 // NewFactory creates a factory for Splunk HEC exporter.
@@ -51,11 +51,11 @@ func createDefaultConfig() configmodels.Exporter {
 		TimeoutSettings: exporterhelper.TimeoutSettings{
 			Timeout: defaultHTTPTimeout,
 		},
-		RetrySettings:      exporterhelper.DefaultRetrySettings(),
-		QueueSettings:      exporterhelper.DefaultQueueSettings(),
-		DisableCompression: false,
-		MaxConnections:     defaultMaxIdleCons,
-		MaxContentLength:   defaultMaxContentLength,
+		RetrySettings:        exporterhelper.DefaultRetrySettings(),
+		QueueSettings:        exporterhelper.DefaultQueueSettings(),
+		DisableCompression:   false,
+		MaxConnections:       defaultMaxIdleCons,
+		MaxContentLengthLogs: defaultMaxContentLengthLogs,
 	}
 }
 
diff --git a/exporter/splunkhecexporter/logdata_to_splunk.go b/exporter/splunkhecexporter/logdata_to_splunk.go
index c02a6eebd6aa..fa3c3091fc0d 100644
--- a/exporter/splunkhecexporter/logdata_to_splunk.go
+++ b/exporter/splunkhecexporter/logdata_to_splunk.go
@@ -86,20 +86,20 @@ func (ld *logDataWrapper) chunkEvents(ctx context.Context, logger *zap.Logger, c
 
 				addToChunk:
 					// The size of an event must be less than or equal to max content length.
-					if config.MaxContentLength > 0 && event.Len() > config.MaxContentLength {
+					if config.MaxContentLengthLogs > 0 && event.Len() > config.MaxContentLengthLogs {
 						select {
 						case <-ctx.Done():
 						case chunkCh <- &eventsChunk{
 							buf:   nil,
 							index: chunk.index,
-							err:   consumererror.Permanent(fmt.Errorf("log event bytes exceed max content length configured (log: %d, max: %d)", event.Len(), config.MaxContentLength))}:
+							err:   consumererror.Permanent(fmt.Errorf("log event bytes exceed max content length configured (log: %d, max: %d)", event.Len(), config.MaxContentLengthLogs))}:
 						}
 						return
 					}
 
 					// Moving the event to chunk.buf if length will be <= max content length.
 					// Max content length <= 0 is interpreted as unbound.
-					if chunk.buf.Len()+event.Len() <= config.MaxContentLength || config.MaxContentLength <= 0 {
+					if chunk.buf.Len()+event.Len() <= config.MaxContentLengthLogs || config.MaxContentLengthLogs <= 0 {
 						// WriteTo() empties and resets buffer event.
 						if _, err := event.WriteTo(chunk.buf); err != nil {
 							select {
diff --git a/exporter/splunkhecexporter/logdata_to_splunk_test.go b/exporter/splunkhecexporter/logdata_to_splunk_test.go
index 5d6649da83e1..57d305e54ba8 100644
--- a/exporter/splunkhecexporter/logdata_to_splunk_test.go
+++ b/exporter/splunkhecexporter/logdata_to_splunk_test.go
@@ -305,7 +305,7 @@ func Test_chunkEvents_MaxContentLength_AllEventsInChunk(t *testing.T) {
 
 	// Chunk max content length to fit all events in 1 chunk.
 	chunkLength := len(events) * max
-	config := Config{MaxContentLength: chunkLength}
+	config := Config{MaxContentLengthLogs: chunkLength}
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -315,7 +315,7 @@ func Test_chunkEvents_MaxContentLength_AllEventsInChunk(t *testing.T) {
 	numChunks := 0
 
 	for chunk := range chunkCh {
-		assert.Nil(t, chunk.err)
+		assert.NoError(t, chunk.err)
 		assert.Len(t, chunk.buf.Bytes(), eventsLength)
 		numChunks++
 	}
@@ -335,7 +335,7 @@ func Test_chunkEvents_MaxContentLength_0(t *testing.T) {
 
 	// Chunk max content length 0 is interpreted as unlimited length.
 	chunkLength := 0
-	config := Config{MaxContentLength: chunkLength}
+	config := Config{MaxContentLengthLogs: chunkLength}
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -345,7 +345,7 @@ func Test_chunkEvents_MaxContentLength_0(t *testing.T) {
 	numChunks := 0
 
 	for chunk := range chunkCh {
-		assert.Nil(t, chunk.err)
+		assert.NoError(t, chunk.err)
 		assert.Len(t, chunk.buf.Bytes(), eventsLength)
 		numChunks++
 	}
@@ -365,7 +365,7 @@ func Test_chunkEvents_MaxContentLength_0_Cancel(t *testing.T) {
 
 	// Chunk max content length 0 is interpreted as unlimited length.
 	chunkLength := 0
-	config := Config{MaxContentLength: chunkLength}
+	config := Config{MaxContentLengthLogs: chunkLength}
 
 	ctx, cancel := context.WithCancel(context.Background())
 
@@ -380,7 +380,7 @@ func Test_chunkEvents_MaxContentLength_0_Cancel(t *testing.T) {
 
 	for chunk := range chunkCh {
 		assert.Nil(t, chunk.buf)
-		assert.Nil(t, chunk.err)
+		assert.NoError(t, chunk.err)
 		numChunks++
 	}
 
@@ -399,7 +399,7 @@ func Test_chunkEvents_MaxContentLength_Negative(t *testing.T) {
 
 	// Negative max content length is interpreted as unlimited length.
 	chunkLength := -3
-	config := Config{MaxContentLength: chunkLength}
+	config := Config{MaxContentLengthLogs: chunkLength}
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -409,7 +409,7 @@ func Test_chunkEvents_MaxContentLength_Negative(t *testing.T) {
 	numChunks := 0
 
 	for chunk := range chunkCh {
-		assert.Nil(t, chunk.err)
+		assert.NoError(t, chunk.err)
 		assert.Len(t, chunk.buf.Bytes(), eventsLength)
 		numChunks++
 	}
@@ -424,7 +424,7 @@ func Test_chunkEvents_MaxContentLength_Small_Error(t *testing.T) {
 
 	// Configuring max content length to be smaller than event lengths.
 	chunkLength := min - 1
-	config := Config{MaxContentLength: chunkLength}
+	config := Config{MaxContentLengthLogs: chunkLength}
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -451,7 +451,7 @@ func Test_chunkEvents_MaxContentLength_Small_Error_Cancel(t *testing.T) {
 
 	// Configuring max content length to be smaller than event lengths.
 	chunkLength := min - 1
-	config := Config{MaxContentLength: chunkLength}
+	config := Config{MaxContentLengthLogs: chunkLength}
 
 	ctx, cancel := context.WithCancel(context.Background())
 
@@ -469,7 +469,7 @@ func Test_chunkEvents_MaxContentLength_Small_Error_Cancel(t *testing.T) {
 			numChunks++
 		}
 		assert.Nil(t, chunk.buf)
-		assert.Nil(t, chunk.err)
+		assert.NoError(t, chunk.err)
 	}
 
 	assert.Equal(t, 0, numChunks)
@@ -482,13 +482,13 @@ func Test_chunkEvents_MaxContentLength_1EventPerChunk(t *testing.T) {
 
 	numEvents := len(events)
 
-	assert.True(t, numEvents > 1, "More than 1 event required")
+	require.True(t, numEvents > 1, "More than 1 event required")
 
-	assert.True(t, minLength >= maxLength/2, "Smallest event >= half largest event required")
+	require.True(t, minLength >= maxLength/2, "Smallest event >= half largest event required")
 
 	// Setting chunk length to have 1 event per chunk.
 	chunkLength := maxLength
-	config := Config{MaxContentLength: chunkLength}
+	config := Config{MaxContentLengthLogs: chunkLength}
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -498,7 +498,7 @@ func Test_chunkEvents_MaxContentLength_1EventPerChunk(t *testing.T) {
 	numChunks := 0
 
 	for chunk := range chunkCh {
-		assert.Nil(t, chunk.err)
+		assert.NoError(t, chunk.err)
 		assert.Len(t, chunk.buf.Bytes(), len(events[numChunks]))
 		numChunks++
 	}
@@ -512,13 +512,13 @@ func Test_chunkEvents_MaxContentLength_1EventPerChunk_Cancel(t *testing.T) {
 
 	min, max, events := jsonEncodeEventsBytes(logs, &Config{})
 
-	assert.True(t, len(events) > 1, "More than 1 event required")
+	require.True(t, len(events) > 1, "More than 1 event required")
 
-	assert.True(t, min >= max/2, "Smallest event >= half largest event required")
+	require.True(t, min >= max/2, "Smallest event >= half largest event required")
 
 	// Setting chunk length to have as many chunks as there are events.
 	chunkLength := max
-	config := Config{MaxContentLength: chunkLength}
+	config := Config{MaxContentLengthLogs: chunkLength}
 
 	ctx, cancel := context.WithCancel(context.Background())
 
@@ -526,7 +526,7 @@ func Test_chunkEvents_MaxContentLength_1EventPerChunk_Cancel(t *testing.T) {
 
 	_, ok := <-chunkCh
 
-	assert.True(t, ok, "Chunk channel open before cancel")
+	require.True(t, ok, "Chunk channel open before cancel")
 
 	cancel()
 
@@ -547,10 +547,10 @@ func Test_chunkEvents_MaxContentLength_2EventsPerChunk(t *testing.T) {
 	numEvents := len(events)
 
 	// Chunk max content length = 2 * max and this condition results in 2 event per chunk.
-	assert.True(t, min >= max/2)
+	require.True(t, min >= max/2)
 
 	chunkLength := 2 * max
-	config := Config{MaxContentLength: chunkLength}
+	config := Config{MaxContentLengthLogs: chunkLength}
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
@@ -562,7 +562,7 @@ func Test_chunkEvents_MaxContentLength_2EventsPerChunk(t *testing.T) {
 	numChunks := 0
 
 	for chunk := range chunkCh {
-		assert.Nil(t, chunk.err)
+		assert.NoError(t, chunk.err)
 		numChunks++
 	}
 

From 6eb3576b55197dace52a626f05bd36014e839ba6 Mon Sep 17 00:00:00 2001
From: Jay Camp <jay.r.camp@gmail.com>
Date: Thu, 11 Mar 2021 17:16:45 -0500
Subject: [PATCH 12/26] wip

---
 exporter/splunkhecexporter/client.go          | 114 ++++++++++----
 .../splunkhecexporter/logdata_to_splunk.go    | 139 +-----------------
 2 files changed, 89 insertions(+), 164 deletions(-)

diff --git a/exporter/splunkhecexporter/client.go b/exporter/splunkhecexporter/client.go
index fee982040491..f6a2284f91e5 100644
--- a/exporter/splunkhecexporter/client.go
+++ b/exporter/splunkhecexporter/client.go
@@ -157,6 +157,72 @@ func (c *client) postEvents(ctx context.Context, events io.Reader, compressed bo
 	return nil
 }
 
+func subLog(ld pdata.Logs, logIdx logIndex) pdata.Logs {
+	if logIdx.zero() {
+		return ld
+	}
+	clone := ld.Clone().InternalRep()
+
+	subset := *clone.Orig
+	subset = subset[logIdx.origIdx:]
+	subset[0].InstrumentationLibraryLogs = subset[0].InstrumentationLibraryLogs[logIdx.instIdx:]
+	subset[0].InstrumentationLibraryLogs[0].Logs = subset[0].InstrumentationLibraryLogs[0].Logs[logIdx.logsIdx:]
+
+	clone.Orig = &subset
+	return pdata.LogsFromInternalRep(clone)
+}
+
+func (c *client) chunk(ctx context.Context, ld pdata.Logs, max int, send func(ctx context.Context, buf *bytes.Buffer) error) (numDroppedLogs int, err error) {
+	// Provide 5000 overflow because it overruns the max content length then trims it block. Hopefully will prevent
+	// extra allocation.
+	buf := bytes.NewBuffer(make([]byte, 0, max+5_000))
+	encoder := json.NewEncoder(buf)
+	checkPoint := 0
+	var logIdx logIndex
+
+	rls := ld.ResourceLogs()
+	for i := 0; i < rls.Len(); i++ {
+		ills := rls.At(i).InstrumentationLibraryLogs()
+		for j := 0; j < ills.Len(); j++ {
+			logs := ills.At(j).Logs()
+			for k := 0; k < logs.Len(); k++ {
+				event := mapLogRecordToSplunkEvent(logs.At(k), c.config, c.logger)
+				if err := encoder.Encode(event); err != nil {
+					numDroppedLogs++
+					continue
+				}
+
+				buf.WriteString("\r\n\r\n")
+
+				if buf.Len() >= max {
+					// May want to check if checkPoint == 0 ?
+					buf.Truncate(checkPoint)
+					if err := send(ctx, buf); err != nil {
+						return numDroppedLogs, consumererror.PartialLogsError(err, subLog(ld, logIdx))
+					}
+					logIdx = logIndex{
+						origIdx: i,
+						instIdx: j,
+						logsIdx: k,
+					}
+					buf.Reset()
+					checkPoint = 0
+				}
+
+				checkPoint = buf.Len()
+			}
+		}
+	}
+
+	if buf.Len() > 0 {
+		if err := send(ctx, buf); err != nil {
+			return numDroppedLogs, consumererror.PartialLogsError(err, subLog(ld, logIdx))
+		}
+	}
+
+	return numDroppedLogs, nil
+}
+
 func (c *client) pushLogData(ctx context.Context, ld pdata.Logs) (numDroppedLogs int, err error) {
 	c.wg.Add(1)
 	defer c.wg.Done()
@@ -168,45 +234,39 @@ func (c *client) pushLogData(ctx context.Context, ld pdata.Logs) (numDroppedLogs
 	gzipWriter.Reset(gzipBuf)
 	defer gzipWriter.Close()
 
-	logs := logDataWrapper{&ld}
-
-	ctx, cancel := context.WithCancel(ctx)
-	defer cancel()
+	// Callback when each chunk is to be sent.
+	sendBatch := func(ctx context.Context, buf *bytes.Buffer) error {
+		compression := false
+		var reader io.Reader
 
-	chunkCh := logs.chunkEvents(ctx, c.logger, c.config)
+		if buf.Len() < 1500 || c.config.DisableCompression {
+			compression = false
+			reader = buf
+		} else {
+			compression = true
 
-	for chunk := range chunkCh {
-		if chunk.err != nil {
-			return logs.numLogs(chunk.index), chunk.err
-		}
+			gzipBuf.Reset()
+			gzipWriter.Reset(gzipBuf)
 
-		if chunk.buf.Len() == 0 {
-			continue
-		}
+			if _, err := io.Copy(gzipWriter, buf); err != nil {
+				return err
+			}
 
-		// Not compressing if compression disabled or payload fit into a single ethernet frame.
-		if chunk.buf.Len() <= 1500 || c.config.DisableCompression {
-			if err = c.postEvents(ctx, chunk.buf, false); err != nil {
-				return logs.numLogs(chunk.index), consumererror.PartialLogsError(err, *logs.subLogs(chunk.index))
+			if err := gzipWriter.Flush(); err != nil {
+				return err
 			}
-			continue
-		}
 
-		if _, err = gzipWriter.Write(chunk.buf.Bytes()); err != nil {
-			return logs.numLogs(chunk.index), consumererror.Permanent(err)
+			reader = gzipBuf
 		}
 
-		gzipWriter.Flush()
-
-		if err = c.postEvents(ctx, gzipBuf, true); err != nil {
-			return logs.numLogs(chunk.index), consumererror.PartialLogsError(err, *logs.subLogs(chunk.index))
+		if err = c.postEvents(ctx, reader, compression); err != nil {
+			return err
 		}
 
-		gzipBuf.Reset()
-		gzipWriter.Reset(gzipBuf)
+		return nil
 	}
 
-	return 0, nil
+	return c.chunk(ctx, ld, c.config.MaxContentLengthLogs, sendBatch)
 }
 
 func encodeBodyEvents(zippers *sync.Pool, evs []*splunk.Event, disableCompression bool) (bodyReader io.Reader, compressed bool, err error) {
diff --git a/exporter/splunkhecexporter/logdata_to_splunk.go b/exporter/splunkhecexporter/logdata_to_splunk.go
index fa3c3091fc0d..abc5c2c36955 100644
--- a/exporter/splunkhecexporter/logdata_to_splunk.go
+++ b/exporter/splunkhecexporter/logdata_to_splunk.go
@@ -15,13 +15,8 @@
 package splunkhecexporter
 
 import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"fmt"
 	"time"
 
-	"go.opentelemetry.io/collector/consumer/consumererror"
 	"go.opentelemetry.io/collector/consumer/pdata"
 	"go.opentelemetry.io/collector/translator/conventions"
 	"go.uber.org/zap"
@@ -29,15 +24,6 @@ import (
 	"github.com/open-telemetry/opentelemetry-collector-contrib/internal/splunk"
 )
 
-// eventsChunk buffers JSON encoded Splunk events.
-// The events are created from LogRecord(s) where one event is created from one LogRecord.
-type eventsChunk struct {
-	buf *bytes.Buffer
-	// The logIndex of the LogRecord of 1st event in buf.
-	index *logIndex
-	err   error
-}
-
 // Composite index of a log record in pdata.Logs.
 type logIndex struct {
 	// Index in orig list (i.e. root parent index).
@@ -48,129 +34,8 @@ type logIndex struct {
 	logsIdx int
 }
 
-type logDataWrapper struct {
-	*pdata.Logs
-}
-
-func (ld *logDataWrapper) chunkEvents(ctx context.Context, logger *zap.Logger, config *Config) chan *eventsChunk {
-	chunkCh := make(chan *eventsChunk)
-
-	go func() {
-		defer close(chunkCh)
-
-		// event buffers a single event.
-		event := new(bytes.Buffer)
-		encoder := json.NewEncoder(event)
-
-		// chunk buffers events up to the max content length.
-		chunk := &eventsChunk{buf: new(bytes.Buffer)}
-
-		rl := ld.ResourceLogs()
-		for i := 0; i < rl.Len(); i++ {
-			ill := rl.At(i).InstrumentationLibraryLogs()
-			for j := 0; j < ill.Len(); j++ {
-				l := ill.At(j).Logs()
-				for k := 0; k < l.Len(); k++ {
-					if chunk.index == nil {
-						chunk.index = &logIndex{origIdx: i, instIdx: j, logsIdx: k}
-					}
-
-					if err := encoder.Encode(mapLogRecordToSplunkEvent(l.At(k), config, logger)); err != nil {
-						select {
-						case <-ctx.Done():
-						case chunkCh <- &eventsChunk{buf: nil, index: chunk.index, err: consumererror.Permanent(err)}:
-						}
-						return
-					}
-					event.WriteString("\r\n\r\n")
-
-				addToChunk:
-					// The size of an event must be less than or equal to max content length.
-					if config.MaxContentLengthLogs > 0 && event.Len() > config.MaxContentLengthLogs {
-						select {
-						case <-ctx.Done():
-						case chunkCh <- &eventsChunk{
-							buf:   nil,
-							index: chunk.index,
-							err:   consumererror.Permanent(fmt.Errorf("log event bytes exceed max content length configured (log: %d, max: %d)", event.Len(), config.MaxContentLengthLogs))}:
-						}
-						return
-					}
-
-					// Moving the event to chunk.buf if length will be <= max content length.
-					// Max content length <= 0 is interpreted as unbound.
-					if chunk.buf.Len()+event.Len() <= config.MaxContentLengthLogs || config.MaxContentLengthLogs <= 0 {
-						// WriteTo() empties and resets buffer event.
-						if _, err := event.WriteTo(chunk.buf); err != nil {
-							select {
-							case <-ctx.Done():
-							case chunkCh <- &eventsChunk{buf: nil, index: chunk.index, err: consumererror.Permanent(err)}:
-							}
-							return
-						}
-						continue
-					}
-
-					select {
-					case <-ctx.Done():
-						return
-					case chunkCh <- chunk:
-					}
-
-					// Creating a new events buffer.
-					chunk = &eventsChunk{buf: new(bytes.Buffer)}
-
-					// Adding remaining event to the new chunk
-					if event.Len() != 0 {
-						chunk.index = &logIndex{origIdx: i, instIdx: j, logsIdx: k}
-						goto addToChunk
-					}
-				}
-			}
-		}
-
-		select {
-		case <-ctx.Done():
-			return
-		case chunkCh <- chunk:
-		}
-	}()
-
-	return chunkCh
-}
-
-func (ld *logDataWrapper) numLogs(from *logIndex) int {
-	count, orig := 0, *ld.InternalRep().Orig
-
-	// Validating logIndex. Invalid index will cause out of range panic.
-	_ = orig[from.origIdx].InstrumentationLibraryLogs[from.instIdx].Logs[from.logsIdx]
-
-	for i := from.origIdx; i < len(orig); i++ {
-		for j, library := range orig[i].InstrumentationLibraryLogs {
-			switch {
-			case i == from.origIdx && j < from.instIdx:
-				continue
-			default:
-				count += len(library.Logs)
-			}
-		}
-	}
-
-	return count - from.logsIdx
-}
-
-func (ld *logDataWrapper) subLogs(from *logIndex) *pdata.Logs {
-	clone := ld.Clone().InternalRep()
-
-	subset := *clone.Orig
-	subset = subset[from.origIdx:]
-	subset[0].InstrumentationLibraryLogs = subset[0].InstrumentationLibraryLogs[from.instIdx:]
-	subset[0].InstrumentationLibraryLogs[0].Logs = subset[0].InstrumentationLibraryLogs[0].Logs[from.logsIdx:]
-
-	clone.Orig = &subset
-	subsetLogs := pdata.LogsFromInternalRep(clone)
-
-	return &subsetLogs
+func (i *logIndex) zero() bool {
+	return i.origIdx == 0 && i.instIdx == 0 && i.logsIdx == 0
 }
 
 func mapLogRecordToSplunkEvent(lr pdata.LogRecord, config *Config, logger *zap.Logger) *splunk.Event {

From 748eec74626a130de21899e8bda23e9fe3ed3216 Mon Sep 17 00:00:00 2001
From: Bremer Jonathan <bjonathan@splunk.com>
Date: Tue, 16 Mar 2021 12:57:23 -0400
Subject: [PATCH 13/26] Handle events buffer dropping last event when buffer
 len >= max content length

---
 exporter/splunkhecexporter/client.go | 34 ++++++++++++++--------------
 1 file changed, 17 insertions(+), 17 deletions(-)

diff --git a/exporter/splunkhecexporter/client.go b/exporter/splunkhecexporter/client.go
index f6a2284f91e5..395bfc83f167 100644
--- a/exporter/splunkhecexporter/client.go
+++ b/exporter/splunkhecexporter/client.go
@@ -173,9 +173,7 @@ func subLog(ld pdata.Logs, logIdx logIndex) pdata.Logs {
 }
 
 func (c *client) chunk(ctx context.Context, ld pdata.Logs, max int, send func(ctx context.Context, buf *bytes.Buffer) error) (numDroppedLogs int, err error) {
-	// Provide 5000 overflow because it overruns the max content length then trims it block. Hopefully will prevent
-	// extra allocation.
-	buf := bytes.NewBuffer(make([]byte, 0, max+5_000))
+	buf := bytes.NewBuffer(make([]byte, 0, max))
 	encoder := json.NewEncoder(buf)
 	checkPoint := 0
 	var logIdx logIndex
@@ -194,22 +192,24 @@ func (c *client) chunk(ctx context.Context, ld pdata.Logs, max int, send func(ct
 
 				buf.WriteString("\r\n\r\n")
 
-				if buf.Len() >= max {
-					// May want to check if checkPoint == 0 ?
-					buf.Truncate(checkPoint)
-					if err := send(ctx, buf); err != nil {
-						return numDroppedLogs, consumererror.PartialLogsError(err, subLog(ld, logIdx))
-					}
-					logIdx = logIndex{
-						origIdx: i,
-						instIdx: j,
-						logsIdx: k,
-					}
-					buf.Reset()
-					checkPoint = 0
+				if buf.Len() < max {
+					checkPoint = buf.Len()
+					continue
+				}
+
+				sending := buf.Next(checkPoint)
+				leaving := buf.Bytes()
+
+				if err := send(ctx, bytes.NewBuffer(sending)); err != nil {
+					return numDroppedLogs, consumererror.PartialLogsError(err, subLog(ld, logIdx))
 				}
 
-				checkPoint = buf.Len()
+				logIdx = logIndex{origIdx: i, instIdx: j, logsIdx: k,}
+
+				buf.Reset()
+				buf.Write(leaving)
+
+				checkPoint = 0
 			}
 		}
 	}

From 1ac7a747960f2769756d61605855274e5f43a13d Mon Sep 17 00:00:00 2001
From: Bremer Jonathan <bjonathan@splunk.com>
Date: Mon, 22 Mar 2021 13:37:17 -0400
Subject: [PATCH 14/26] Refactor pushLogData

---
 exporter/splunkhecexporter/client.go          | 139 ++--
 exporter/splunkhecexporter/client_test.go     | 334 +++++++---
 exporter/splunkhecexporter/config.go          |   7 +-
 exporter/splunkhecexporter/config_test.go     |  15 +-
 exporter/splunkhecexporter/exporter_test.go   |  26 +-
 exporter/splunkhecexporter/factory.go         |  20 +-
 .../splunkhecexporter/logdata_to_splunk.go    |   8 +-
 .../logdata_to_splunk_test.go                 | 613 ++----------------
 8 files changed, 428 insertions(+), 734 deletions(-)

diff --git a/exporter/splunkhecexporter/client.go b/exporter/splunkhecexporter/client.go
index 395bfc83f167..a651195a7b67 100644
--- a/exporter/splunkhecexporter/client.go
+++ b/exporter/splunkhecexporter/client.go
@@ -57,7 +57,7 @@ func (c *client) pushMetricsData(
 		return numDroppedTimeseries, nil
 	}
 
-	body, compressed, err := encodeBody(&c.zippers, splunkDataPoints, c.config.DisableCompression)
+	body, compressed, err := encodeBody(&c.zippers, splunkDataPoints, c.config.DisableCompression, c.config.MinContentLengthCompression)
 	if err != nil {
 		return numMetricPoint(md), consumererror.Permanent(err)
 	}
@@ -116,7 +116,7 @@ func (c *client) pushTraceData(
 }
 
 func (c *client) sendSplunkEvents(ctx context.Context, splunkEvents []*splunk.Event) error {
-	body, compressed, err := encodeBodyEvents(&c.zippers, splunkEvents, c.config.DisableCompression)
+	body, compressed, err := encodeBodyEvents(&c.zippers, splunkEvents, c.config.DisableCompression, c.config.MinContentLengthCompression)
 	if err != nil {
 		return consumererror.Permanent(err)
 	}
@@ -157,26 +157,37 @@ func (c *client) postEvents(ctx context.Context, events io.Reader, compressed bo
 	return nil
 }
 
-func subLog(ld pdata.Logs, logIdx logIndex) pdata.Logs {
+func subLogs(ld *pdata.Logs, logIdx *logIndex) *pdata.Logs {
 	if logIdx.zero() {
 		return ld
 	}
 	clone := ld.Clone().InternalRep()
 
 	subset := *clone.Orig
-	subset = subset[logIdx.origIdx:]
-	subset[0].InstrumentationLibraryLogs = subset[0].InstrumentationLibraryLogs[logIdx.instIdx:]
-	subset[0].InstrumentationLibraryLogs[0].Logs = subset[0].InstrumentationLibraryLogs[0].Logs[logIdx.logsIdx:]
+	subset = subset[logIdx.resourceIdx:]
+	subset[0].InstrumentationLibraryLogs = subset[0].InstrumentationLibraryLogs[logIdx.libraryIdx:]
+	subset[0].InstrumentationLibraryLogs[0].Logs = subset[0].InstrumentationLibraryLogs[0].Logs[logIdx.recordIdx:]
 
 	clone.Orig = &subset
-	return pdata.LogsFromInternalRep(clone)
+	logs := pdata.LogsFromInternalRep(clone)
+	return &logs
 }
 
-func (c *client) chunk(ctx context.Context, ld pdata.Logs, max int, send func(ctx context.Context, buf *bytes.Buffer) error) (numDroppedLogs int, err error) {
-	buf := bytes.NewBuffer(make([]byte, 0, max))
-	encoder := json.NewEncoder(buf)
-	checkPoint := 0
-	var logIdx logIndex
+func (c *client) sentLogBatch(ctx context.Context, ld pdata.Logs, send func(context.Context, *bytes.Buffer) error) (numDroppedLogs int, err error) {
+	var submax int
+	var index *logIndex
+	var permanentErrors []error
+	var batch *bytes.Buffer
+
+	// Provide 5000 overflow because it overruns the max content length then trims it block. Hopefully will prevent extra allocation.
+	if c.config.MaxContentLengthLogs > 0 {
+		batch = bytes.NewBuffer(make([]byte, 0, c.config.MaxContentLengthLogs+5_000))
+	} else {
+		batch = bytes.NewBuffer(make([]byte, 0))
+	}
+	encoder := json.NewEncoder(batch)
+
+	overflow := bytes.NewBuffer(make([]byte, 0, 5000))
 
 	rls := ld.ResourceLogs()
 	for i := 0; i < rls.Len(); i++ {
@@ -184,43 +195,56 @@ func (c *client) chunk(ctx context.Context, ld pdata.Logs, max int, send func(ct
 		for j := 0; j < ills.Len(); j++ {
 			logs := ills.At(j).Logs()
 			for k := 0; k < logs.Len(); k++ {
+				if index == nil {
+					index = &logIndex{resourceIdx: i, libraryIdx: j, recordIdx: k}
+				}
+
 				event := mapLogRecordToSplunkEvent(logs.At(k), c.config, c.logger)
-				if err := encoder.Encode(event); err != nil {
-					numDroppedLogs++
+				if err = encoder.Encode(event); err != nil {
+					permanentErrors = append(permanentErrors, consumererror.Permanent(err))
 					continue
 				}
+				batch.WriteString("\r\n\r\n")
 
-				buf.WriteString("\r\n\r\n")
-
-				if buf.Len() < max {
-					checkPoint = buf.Len()
+				// Consistent with ContentLength in http.Request, MaxContentLengthLogs value of 0 indicates length unknown (i.e. unbound).
+				if c.config.MaxContentLengthLogs == 0 || batch.Len() <= int(c.config.MaxContentLengthLogs) {
+					submax = batch.Len()
 					continue
 				}
 
-				sending := buf.Next(checkPoint)
-				leaving := buf.Bytes()
-
-				if err := send(ctx, bytes.NewBuffer(sending)); err != nil {
-					return numDroppedLogs, consumererror.PartialLogsError(err, subLog(ld, logIdx))
+				overflow.Reset()
+				if c.config.MaxContentLengthLogs > 0 {
+					if over := batch.Len() - submax; over <= int(c.config.MaxContentLengthLogs) {
+						overflow.Write(batch.Bytes()[submax:batch.Len()])
+					} else {
+						err = fmt.Errorf("log event too large (configured max content length: %d bytes, event: %d bytes)", c.config.MaxContentLengthLogs, over)
+						permanentErrors = append(permanentErrors, consumererror.Permanent(err))
+					}
 				}
 
-				logIdx = logIndex{origIdx: i, instIdx: j, logsIdx: k,}
-
-				buf.Reset()
-				buf.Write(leaving)
+				batch.Truncate(submax)
+				if batch.Len() > 0 {
+					if err = send(ctx, batch); err != nil {
+						dropped := subLogs(&ld, index)
+						return dropped.LogRecordCount(), consumererror.PartialLogsError(err, *dropped)
+					}
+				}
+				batch.Reset()
+				overflow.WriteTo(batch)
 
-				checkPoint = 0
+				index, submax = nil, batch.Len()
 			}
 		}
 	}
 
-	if buf.Len() > 0 {
-		if err := send(ctx, buf); err != nil {
-			return numDroppedLogs, consumererror.PartialLogsError(err, subLog(ld, logIdx))
+	if batch.Len() > 0 {
+		if err = send(ctx, batch); err != nil {
+			dropped := subLogs(&ld, index)
+			return dropped.LogRecordCount(), consumererror.PartialLogsError(err, *dropped)
 		}
 	}
 
-	return numDroppedLogs, nil
+	return len(permanentErrors), consumererror.CombineErrors(permanentErrors)
 }
 
 func (c *client) pushLogData(ctx context.Context, ld pdata.Logs) (numDroppedLogs int, err error) {
@@ -230,46 +254,41 @@ func (c *client) pushLogData(ctx context.Context, ld pdata.Logs) (numDroppedLogs
 	gzipWriter := c.zippers.Get().(*gzip.Writer)
 	defer c.zippers.Put(gzipWriter)
 
-	gzipBuf := bytes.NewBuffer(make([]byte, 0, c.config.MaxContentLengthLogs))
+	var gzipBuf *bytes.Buffer
+	if c.config.MaxContentLengthLogs > 0 {
+		gzipBuf = bytes.NewBuffer(make([]byte, 0, c.config.MaxContentLengthLogs))
+	} else {
+		gzipBuf = bytes.NewBuffer(make([]byte, 0))
+	}
 	gzipWriter.Reset(gzipBuf)
 	defer gzipWriter.Close()
 
-	// Callback when each chunk is to be sent.
-	sendBatch := func(ctx context.Context, buf *bytes.Buffer) error {
-		compression := false
-		var reader io.Reader
-
-		if buf.Len() < 1500 || c.config.DisableCompression {
-			compression = false
-			reader = buf
-		} else {
-			compression = true
+	// Callback when each batch is to be sent.
+	send := func(ctx context.Context, buf *bytes.Buffer) (err error) {
+		compression := buf.Len() >= int(c.config.MinContentLengthCompression) && !c.config.DisableCompression
 
+		if compression {
 			gzipBuf.Reset()
 			gzipWriter.Reset(gzipBuf)
 
-			if _, err := io.Copy(gzipWriter, buf); err != nil {
-				return err
+			if _, err = io.Copy(gzipWriter, buf); err != nil {
+				return
 			}
 
-			if err := gzipWriter.Flush(); err != nil {
-				return err
+			if err = gzipWriter.Flush(); err != nil {
+				return
 			}
 
-			reader = gzipBuf
-		}
-
-		if err = c.postEvents(ctx, reader, compression); err != nil {
-			return err
+			return c.postEvents(ctx, gzipBuf, compression)
 		}
 
-		return nil
+		return c.postEvents(ctx, buf, compression)
 	}
 
-	return c.chunk(ctx, ld, c.config.MaxContentLengthLogs, sendBatch)
+	return c.sentLogBatch(ctx, ld, send)
 }
 
-func encodeBodyEvents(zippers *sync.Pool, evs []*splunk.Event, disableCompression bool) (bodyReader io.Reader, compressed bool, err error) {
+func encodeBodyEvents(zippers *sync.Pool, evs []*splunk.Event, disableCompression bool, minContentLengthCompression uint) (bodyReader io.Reader, compressed bool, err error) {
 	buf := new(bytes.Buffer)
 	encoder := json.NewEncoder(buf)
 	for _, e := range evs {
@@ -279,10 +298,10 @@ func encodeBodyEvents(zippers *sync.Pool, evs []*splunk.Event, disableCompressio
 		}
 		buf.WriteString("\r\n\r\n")
 	}
-	return getReader(zippers, buf, disableCompression)
+	return getReader(zippers, buf, disableCompression, minContentLengthCompression)
 }
 
-func encodeBody(zippers *sync.Pool, dps []*splunk.Event, disableCompression bool) (bodyReader io.Reader, compressed bool, err error) {
+func encodeBody(zippers *sync.Pool, dps []*splunk.Event, disableCompression bool, minContentLengthCompression uint) (bodyReader io.Reader, compressed bool, err error) {
 	buf := new(bytes.Buffer)
 	encoder := json.NewEncoder(buf)
 	for _, e := range dps {
@@ -292,13 +311,13 @@ func encodeBody(zippers *sync.Pool, dps []*splunk.Event, disableCompression bool
 		}
 		buf.WriteString("\r\n\r\n")
 	}
-	return getReader(zippers, buf, disableCompression)
+	return getReader(zippers, buf, disableCompression, minContentLengthCompression)
 }
 
 // avoid attempting to compress things that fit into a single ethernet frame
-func getReader(zippers *sync.Pool, b *bytes.Buffer, disableCompression bool) (io.Reader, bool, error) {
+func getReader(zippers *sync.Pool, b *bytes.Buffer, disableCompression bool, minContentLengthCompression uint) (io.Reader, bool, error) {
 	var err error
-	if !disableCompression && b.Len() > 1500 {
+	if !disableCompression && b.Len() > int(minContentLengthCompression) {
 		buf := new(bytes.Buffer)
 		w := zippers.Get().(*gzip.Writer)
 		defer zippers.Put(w)
diff --git a/exporter/splunkhecexporter/client_test.go b/exporter/splunkhecexporter/client_test.go
index cc7c3d161582..386d0bd580c4 100644
--- a/exporter/splunkhecexporter/client_test.go
+++ b/exporter/splunkhecexporter/client_test.go
@@ -17,6 +17,7 @@ import (
 	"compress/gzip"
 	"context"
 	"errors"
+	"fmt"
 	"io/ioutil"
 	"math"
 	"net"
@@ -26,6 +27,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/stretchr/testify/require"
+
 	"github.com/stretchr/testify/assert"
 	"go.opentelemetry.io/collector/component"
 	"go.opentelemetry.io/collector/component/componenttest"
@@ -95,44 +98,52 @@ func createTraceData(numberOfTraces int) pdata.Traces {
 	return traces
 }
 
-func createLogData(numberOfLogs int) pdata.Logs {
+func createLogData(numResources int, numLibraries int, numRecords int) pdata.Logs {
 	logs := pdata.NewLogs()
-	logs.ResourceLogs().Resize(1)
-	rl := logs.ResourceLogs().At(0)
-	rl.InstrumentationLibraryLogs().Resize(1)
-	ill := rl.InstrumentationLibraryLogs().At(0)
-
-	for i := 0; i < numberOfLogs; i++ {
-		ts := pdata.TimestampUnixNano(int64(i) * time.Millisecond.Nanoseconds())
-		logRecord := pdata.NewLogRecord()
-		logRecord.Body().SetStringVal("mylog")
-		logRecord.Attributes().InsertString(conventions.AttributeServiceName, "myapp")
-		logRecord.Attributes().InsertString(splunk.SourcetypeLabel, "myapp-type")
-		logRecord.Attributes().InsertString(splunk.IndexLabel, "myindex")
-		logRecord.Attributes().InsertString(conventions.AttributeHostName, "myhost")
-		logRecord.Attributes().InsertString("custom", "custom")
-		logRecord.SetTimestamp(ts)
-
-		ill.Logs().Append(logRecord)
+	logs.ResourceLogs().Resize(numResources)
+
+	for i := 0; i < numResources; i++ {
+		rl := logs.ResourceLogs().At(i)
+		rl.InstrumentationLibraryLogs().Resize(numLibraries)
+		for j := 0; j < numLibraries; j++ {
+			ill := rl.InstrumentationLibraryLogs().At(j)
+			for k := 0; k < numRecords; k++ {
+				ts := pdata.TimestampUnixNano(int64(k) * time.Millisecond.Nanoseconds())
+				logRecord := pdata.NewLogRecord()
+				logRecord.SetName(fmt.Sprintf("%d_%d_%d", i, j, k))
+				logRecord.Body().SetStringVal("mylog")
+				logRecord.Attributes().InsertString(conventions.AttributeServiceName, "myapp")
+				logRecord.Attributes().InsertString(splunk.SourcetypeLabel, "myapp-type")
+				logRecord.Attributes().InsertString(splunk.IndexLabel, "myindex")
+				logRecord.Attributes().InsertString(conventions.AttributeHostName, "myhost")
+				logRecord.Attributes().InsertString("custom", "custom")
+				logRecord.SetTimestamp(ts)
+
+				ill.Logs().Append(logRecord)
+			}
+		}
 	}
 
 	return logs
 }
 
 type CapturingData struct {
-	testing          *testing.T
-	receivedRequest  chan string
-	statusCode       int
-	checkCompression bool
+	testing                     *testing.T
+	receivedRequest             chan string
+	statusCode                  int
+	checkCompression            bool
+	minContentLengthCompression uint
 }
 
 func (c *CapturingData) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	body, err := ioutil.ReadAll(r.Body)
+
 	if c.checkCompression {
-		if r.Header.Get("Content-Encoding") != "gzip" {
+		if len(body) > int(c.minContentLengthCompression) && r.Header.Get("Content-Encoding") != "gzip" {
 			c.testing.Fatal("No compression")
 		}
 	}
-	body, err := ioutil.ReadAll(r.Body)
+
 	if err != nil {
 		panic(err)
 	}
@@ -143,18 +154,10 @@ func (c *CapturingData) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 }
 
 func runMetricsExport(disableCompression bool, numberOfDataPoints int, t *testing.T) (string, error) {
-	receivedRequest := make(chan string)
-	capture := CapturingData{testing: t, receivedRequest: receivedRequest, statusCode: 200, checkCompression: !disableCompression}
 	listener, err := net.Listen("tcp", "127.0.0.1:0")
 	if err != nil {
 		panic(err)
 	}
-	s := &http.Server{
-		Handler: &capture,
-	}
-	go func() {
-		panic(s.Serve(listener))
-	}()
 
 	factory := NewFactory()
 	cfg := factory.CreateDefaultConfig().(*Config)
@@ -162,6 +165,15 @@ func runMetricsExport(disableCompression bool, numberOfDataPoints int, t *testin
 	cfg.DisableCompression = disableCompression
 	cfg.Token = "1234-1234"
 
+	receivedRequest := make(chan string)
+	capture := CapturingData{testing: t, receivedRequest: receivedRequest, statusCode: 200, checkCompression: !cfg.DisableCompression, minContentLengthCompression: cfg.MinContentLengthCompression}
+	s := &http.Server{
+		Handler: &capture,
+	}
+	go func() {
+		panic(s.Serve(listener))
+	}()
+
 	params := component.ExporterCreateParams{Logger: zap.NewNop()}
 	exporter, err := factory.CreateMetricsExporter(context.Background(), params, cfg)
 	assert.NoError(t, err)
@@ -181,18 +193,10 @@ func runMetricsExport(disableCompression bool, numberOfDataPoints int, t *testin
 }
 
 func runTraceExport(disableCompression bool, numberOfTraces int, t *testing.T) (string, error) {
-	receivedRequest := make(chan string)
-	capture := CapturingData{testing: t, receivedRequest: receivedRequest, statusCode: 200, checkCompression: !disableCompression}
 	listener, err := net.Listen("tcp", "127.0.0.1:0")
 	if err != nil {
 		panic(err)
 	}
-	s := &http.Server{
-		Handler: &capture,
-	}
-	go func() {
-		panic(s.Serve(listener))
-	}()
 
 	factory := NewFactory()
 	cfg := factory.CreateDefaultConfig().(*Config)
@@ -200,6 +204,15 @@ func runTraceExport(disableCompression bool, numberOfTraces int, t *testing.T) (
 	cfg.DisableCompression = disableCompression
 	cfg.Token = "1234-1234"
 
+	receivedRequest := make(chan string)
+	capture := CapturingData{testing: t, receivedRequest: receivedRequest, statusCode: 200, checkCompression: !cfg.DisableCompression, minContentLengthCompression: cfg.MinContentLengthCompression}
+	s := &http.Server{
+		Handler: &capture,
+	}
+	go func() {
+		panic(s.Serve(listener))
+	}()
+
 	params := component.ExporterCreateParams{Logger: zap.NewNop()}
 	exporter, err := factory.CreateTracesExporter(context.Background(), params, cfg)
 	assert.NoError(t, err)
@@ -218,13 +231,17 @@ func runTraceExport(disableCompression bool, numberOfTraces int, t *testing.T) (
 	}
 }
 
-func runLogExport(disableCompression bool, numberOfLogs int, t *testing.T) (string, error) {
-	receivedRequest := make(chan string)
-	capture := CapturingData{testing: t, receivedRequest: receivedRequest, statusCode: 200, checkCompression: !disableCompression}
+func runLogExport(cfg *Config, ld pdata.Logs, t *testing.T) ([]string, error) {
 	listener, err := net.Listen("tcp", "127.0.0.1:0")
 	if err != nil {
 		panic(err)
 	}
+
+	cfg.Endpoint = "http://" + listener.Addr().String() + "/services/collector"
+	cfg.Token = "1234-1234"
+
+	receivedRequest := make(chan string)
+	capture := CapturingData{testing: t, receivedRequest: receivedRequest, statusCode: 200, checkCompression: !cfg.DisableCompression, minContentLengthCompression: cfg.MinContentLengthCompression}
 	s := &http.Server{
 		Handler: &capture,
 	}
@@ -232,27 +249,26 @@ func runLogExport(disableCompression bool, numberOfLogs int, t *testing.T) (stri
 		panic(s.Serve(listener))
 	}()
 
-	factory := NewFactory()
-	cfg := factory.CreateDefaultConfig().(*Config)
-	cfg.Endpoint = "http://" + listener.Addr().String() + "/services/collector"
-	cfg.DisableCompression = disableCompression
-	cfg.Token = "1234-1234"
-
 	params := component.ExporterCreateParams{Logger: zap.NewNop()}
-	exporter, err := factory.CreateLogsExporter(context.Background(), params, cfg)
+	exporter, err := NewFactory().CreateLogsExporter(context.Background(), params, cfg)
 	assert.NoError(t, err)
 	assert.NoError(t, exporter.Start(context.Background(), componenttest.NewNopHost()))
 	defer exporter.Shutdown(context.Background())
 
-	ld := createLogData(numberOfLogs)
-
 	err = exporter.ConsumeLogs(context.Background(), ld)
 	assert.NoError(t, err)
-	select {
-	case request := <-receivedRequest:
-		return request, nil
-	case <-time.After(1 * time.Second):
-		return "", errors.New("timeout")
+
+	var requests []string
+	for {
+		select {
+		case request := <-receivedRequest:
+			requests = append(requests, request)
+		case <-time.After(1 * time.Second):
+			if len(requests) == 0 {
+				err = errors.New("timeout")
+			}
+			return requests, err
+		}
 	}
 }
 
@@ -269,15 +285,106 @@ func TestReceiveTraces(t *testing.T) {
 }
 
 func TestReceiveLogs(t *testing.T) {
-	actual, err := runLogExport(true, 3, t)
-	assert.NoError(t, err)
-	expected := `{"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom"}}`
-	expected += "\n\r\n\r\n"
-	expected += `{"time":0.001,"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom"}}`
-	expected += "\n\r\n\r\n"
-	expected += `{"time":0.002,"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom"}}`
-	expected += "\n\r\n\r\n"
-	assert.Equal(t, expected, actual)
+	type wantType struct {
+		batches    []string
+		numBatches int
+	}
+
+	tests := []struct {
+		name func(bool) string
+		conf func(bool) *Config
+		logs pdata.Logs
+		want wantType
+	}{
+		{
+			name: func(disableCompression bool) string {
+				return "COMPRESSION " + map[bool]string{true: "DISABLED ", false: "ENABLED "}[disableCompression] +
+					"all log events in payload when max content length unknown (configured max content length 0)"
+			},
+			logs: createLogData(1, 1, 4),
+			conf: func(disableCompression bool) *Config {
+				cfg := NewFactory().CreateDefaultConfig().(*Config)
+				cfg.DisableCompression = disableCompression
+				cfg.MinContentLengthCompression = 4 // Given the 4 logs, 4 bytes minimum triggers compression when enable.
+				cfg.MaxContentLengthLogs = 0
+				return cfg
+			},
+			want: wantType{
+				batches: []string{
+					`{"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom"}}` + "\n\r\n\r\n" +
+						`{"time":0.001,"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom"}}` + "\n\r\n\r\n" +
+						`{"time":0.002,"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom"}}` + "\n\r\n\r\n" +
+						`{"time":0.003,"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom"}}` + "\n\r\n\r\n",
+				},
+				numBatches: 1,
+			},
+		},
+		{
+			name: func(disableCompression bool) string {
+				return "COMPRESSION " + map[bool]string{true: "DISABLED ", false: "ENABLED "}[disableCompression] +
+					"1 log event per payload (configured max content length is same as event size)"
+			},
+			logs: createLogData(1, 1, 4),
+			conf: func(disableCompression bool) *Config {
+				cfg := NewFactory().CreateDefaultConfig().(*Config)
+				cfg.DisableCompression = disableCompression
+				cfg.MinContentLengthCompression = 4 // Given the 4 logs, 4 bytes minimum triggers compression when enable.
+				cfg.MaxContentLengthLogs = 150
+				return cfg
+			},
+			want: wantType{
+				batches: []string{
+					`{"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom"}}` + "\n\r\n\r\n",
+					`{"time":0.001,"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom"}}` + "\n\r\n\r\n",
+					`{"time":0.002,"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom"}}` + "\n\r\n\r\n",
+					`{"time":0.003,"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom"}}` + "\n\r\n\r\n",
+				},
+				numBatches: 4,
+			},
+		},
+		{
+			name: func(disableCompression bool) string {
+				return "COMPRESSION " + map[bool]string{true: "DISABLED ", false: "ENABLED "}[disableCompression] +
+					"2 log events per payload (configured max content length is twice event size)"
+			},
+			logs: createLogData(1, 1, 4),
+			conf: func(disableCompression bool) *Config {
+				cfg := NewFactory().CreateDefaultConfig().(*Config)
+				cfg.DisableCompression = disableCompression
+				cfg.MinContentLengthCompression = 4 // Given the 4 logs, 4 bytes minimum triggers compression when enable.
+				cfg.MaxContentLengthLogs = 300
+				return cfg
+			},
+			want: wantType{
+				batches: []string{
+					`{"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom"}}` + "\n\r\n\r\n" +
+						`{"time":0.001,"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom"}}` + "\n\r\n\r\n",
+					`{"time":0.002,"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom"}}` + "\n\r\n\r\n" +
+						`{"time":0.003,"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom"}}` + "\n\r\n\r\n",
+				},
+				numBatches: 2,
+			},
+		},
+	}
+
+	for _, test := range tests {
+		for _, disabled := range []bool{true, false} {
+			t.Run(test.name(disabled), func(t *testing.T) {
+				got, err := runLogExport(test.conf(disabled), test.logs, t)
+				require.NoError(t, err)
+
+				require.Len(t, got, test.want.numBatches)
+
+				for i := 0; i < test.want.numBatches; i++ {
+					require.NotZero(t, got[i])
+
+					if disabled {
+						assert.Equal(t, test.want.batches[i], got[i])
+					}
+				}
+			})
+		}
+	}
 }
 
 func TestReceiveMetrics(t *testing.T) {
@@ -298,12 +405,6 @@ func TestReceiveTracesWithCompression(t *testing.T) {
 	assert.NotEqual(t, "", request)
 }
 
-func TestReceiveLogsWithCompression(t *testing.T) {
-	request, err := runLogExport(false, 1000, t)
-	assert.NoError(t, err)
-	assert.NotEqual(t, "", request)
-}
-
 func TestReceiveMetricsWithCompression(t *testing.T) {
 	request, err := runMetricsExport(false, 1000, t)
 	assert.NoError(t, err)
@@ -358,7 +459,9 @@ func TestInvalidTraces(t *testing.T) {
 }
 
 func TestInvalidLogs(t *testing.T) {
-	_, err := runLogExport(false, 0, t)
+	config := NewFactory().CreateDefaultConfig().(*Config)
+	config.DisableCompression = false
+	_, err := runLogExport(config, createLogData(1, 1, 0), t)
 	assert.Error(t, err)
 }
 
@@ -406,7 +509,7 @@ func TestInvalidJson(t *testing.T) {
 		},
 		nil,
 	}
-	reader, _, err := encodeBodyEvents(&syncPool, evs, false)
+	reader, _, err := encodeBodyEvents(&syncPool, evs, false, defaultMinContentLengthCompression)
 	assert.Error(t, err, reader)
 }
 
@@ -478,15 +581,88 @@ func Test_pushLogData_PostError(t *testing.T) {
 		zippers: sync.Pool{New: func() interface{} {
 			return gzip.NewWriter(nil)
 		}},
-		config: &Config{DisableCompression: false},
+		config: NewFactory().CreateDefaultConfig().(*Config),
 	}
 
 	numLogs := 1500
-	logs := createLogData(numLogs)
+	logs := createLogData(1, 1, numLogs)
 
-	numDroppedLogs, err := c.pushLogData(context.Background(), logs)
-	if assert.Error(t, err) {
-		assert.IsType(t, consumererror.PartialError{}, err)
+	// Given 1500 logs, 1024 bytes is small enough to trigger compression when compression enable.
+	c.config.MinContentLengthCompression = 1024
+
+	for _, disable := range []bool{true, false} {
+		c.config.DisableCompression = disable
+
+		numDroppedLogs, err := c.pushLogData(context.Background(), logs)
+		if assert.Error(t, err) {
+			assert.IsType(t, consumererror.PartialError{}, err)
+			assert.Equal(t, numLogs, numDroppedLogs)
+		}
+	}
+}
+
+func Test_pushLogData_Small_MaxContentLength(t *testing.T) {
+	c := client{
+		zippers: sync.Pool{New: func() interface{} {
+			return gzip.NewWriter(nil)
+		}},
+		config: NewFactory().CreateDefaultConfig().(*Config),
+	}
+	length1byte := uint(1)
+	c.config.MinContentLengthCompression = length1byte
+	c.config.MaxContentLengthLogs = length1byte
+
+	numLogs := 4
+	logs := createLogData(1, 1, numLogs)
+
+	for _, disable := range []bool{true, false} {
+		c.config.DisableCompression = disable
+
+		numDroppedLogs, err := c.pushLogData(context.Background(), logs)
+		require.Error(t, err)
+
+		assert.True(t, consumererror.IsPermanent(err))
+		assert.Contains(t, err.Error(), "log event too large")
 		assert.Equal(t, numLogs, numDroppedLogs)
 	}
 }
+
+func TestSubLogs(t *testing.T) {
+	// Creating 12 logs (2 resources x 2 libraries x 3 records)
+	logs := createLogData(2, 2, 3)
+
+	// Logs subset from leftmost index (resource 0, library 0, record 0).
+	_0_0_0 := &logIndex{resourceIdx: 0, libraryIdx: 0, recordIdx: 0}
+	got := subLogs(&logs, _0_0_0)
+
+	// Number of logs in subset should equal original logs.
+	assert.Equal(t, logs.LogRecordCount(), got.LogRecordCount())
+
+	orig := *got.InternalRep().Orig
+	// The name of the leftmost log record should be 0_0_0.
+	assert.Equal(t, "0_0_0", orig[0].InstrumentationLibraryLogs[0].Logs[0].Name)
+	// The name of the leftmost log record should be 1_1_2.
+	assert.Equal(t, "1_1_2", orig[1].InstrumentationLibraryLogs[1].Logs[2].Name)
+
+	// Logs subset from some mid index (resource 0, library 1, log 2).
+	_0_1_2 := &logIndex{resourceIdx: 0, libraryIdx: 1, recordIdx: 2}
+	got = subLogs(&logs, _0_1_2)
+
+	assert.Equal(t, 7, got.LogRecordCount())
+
+	orig = *got.InternalRep().Orig
+	// The name of the leftmost log record should be 0_1_2.
+	assert.Equal(t, "0_1_2", orig[0].InstrumentationLibraryLogs[0].Logs[0].Name)
+	// The name of the rightmost log record should be 1_1_2.
+	assert.Equal(t, "1_1_2", orig[1].InstrumentationLibraryLogs[1].Logs[2].Name)
+
+	// Logs subset from some rightmost index (resource 0, library 1, log 2).
+	_1_1_2 := &logIndex{resourceIdx: 1, libraryIdx: 1, recordIdx: 2}
+	got = subLogs(&logs, _1_1_2)
+
+	assert.Equal(t, 1, got.LogRecordCount())
+
+	orig = *got.InternalRep().Orig
+	// The name of the sole log record should be 1_1_2.
+	assert.Equal(t, "1_1_2", orig[0].InstrumentationLibraryLogs[0].Logs[0].Name)
+}
diff --git a/exporter/splunkhecexporter/config.go b/exporter/splunkhecexporter/config.go
index dc8dc89cfcf3..a2f7226a9503 100644
--- a/exporter/splunkhecexporter/config.go
+++ b/exporter/splunkhecexporter/config.go
@@ -58,11 +58,14 @@ type Config struct {
 	// Disable GZip compression. Defaults to false.
 	DisableCompression bool `mapstructure:"disable_compression"`
 
+	// Minimum content length in bytes to compress. Defaults to 1500 bytes (Maximum Transmission Unit of an ethernet frame).
+	MinContentLengthCompression uint `mapstructure:"min_content_length_compression"`
+
 	// insecure_skip_verify skips checking the certificate of the HEC endpoint when sending data over HTTPS. Defaults to false.
 	InsecureSkipVerify bool `mapstructure:"insecure_skip_verify"`
 
-	// MaxContentLengthLogs is the maximum log data size per HTTP post limited to 1048576 bytes (1Mib). Defaults to 1048576.
-	MaxContentLengthLogs int `mapstructure:"max_content_length_logs"`
+	// MaxContentLengthLogs is the maximum log data size in bytes per HTTP post. Defaults to the backend limit of 1048576 bytes (1MiB).
+	MaxContentLengthLogs uint `mapstructure:"max_content_length_logs"`
 }
 
 func (cfg *Config) getOptionsFromConfig() (*exporterOptions, error) {
diff --git a/exporter/splunkhecexporter/config_test.go b/exporter/splunkhecexporter/config_test.go
index e2e81c470d9f..f306b59b1df7 100644
--- a/exporter/splunkhecexporter/config_test.go
+++ b/exporter/splunkhecexporter/config_test.go
@@ -58,13 +58,14 @@ func TestLoadConfig(t *testing.T) {
 			TypeVal: configmodels.Type(typeStr),
 			NameVal: expectedName,
 		},
-		Token:                "00000000-0000-0000-0000-0000000000000",
-		Endpoint:             "https://splunk:8088/services/collector",
-		Source:               "otel",
-		SourceType:           "otel",
-		Index:                "metrics",
-		MaxConnections:       100,
-		MaxContentLengthLogs: 1024 * 1024,
+		Token:                       "00000000-0000-0000-0000-0000000000000",
+		Endpoint:                    "https://splunk:8088/services/collector",
+		Source:                      "otel",
+		SourceType:                  "otel",
+		Index:                       "metrics",
+		MaxConnections:              100,
+		MinContentLengthCompression: defaultMinContentLengthCompression,
+		MaxContentLengthLogs:        defaultMaxContentLengthLogs,
 		TimeoutSettings: exporterhelper.TimeoutSettings{
 			Timeout: 10 * time.Second,
 		},
diff --git a/exporter/splunkhecexporter/exporter_test.go b/exporter/splunkhecexporter/exporter_test.go
index 8e458ad89c6b..1025e67b4f93 100644
--- a/exporter/splunkhecexporter/exporter_test.go
+++ b/exporter/splunkhecexporter/exporter_test.go
@@ -142,12 +142,13 @@ func TestConsumeMetricsData(t *testing.T) {
 				url:   serverURL,
 				token: "1234",
 			}
-			config := &Config{
-				Source:     "test",
-				SourceType: "test_type",
-				Token:      "1234",
-				Index:      "test_index",
-			}
+
+			config := NewFactory().CreateDefaultConfig().(*Config)
+			config.Source = "test"
+			config.SourceType = "test_type"
+			config.Token = "1234"
+			config.Index = "test_index"
+
 			sender := buildClient(options, config, zap.NewNop())
 
 			md := internaldata.OCToMetrics(tt.md)
@@ -290,12 +291,13 @@ func TestConsumeLogsData(t *testing.T) {
 				url:   serverURL,
 				token: "1234",
 			}
-			config := &Config{
-				Source:     "test",
-				SourceType: "test_type",
-				Token:      "1234",
-				Index:      "test_index",
-			}
+
+			config := NewFactory().CreateDefaultConfig().(*Config)
+			config.Source = "test"
+			config.SourceType = "test_type"
+			config.Token = "1234"
+			config.Index = "test_index"
+
 			sender := buildClient(options, config, zap.NewNop())
 
 			numDroppedLogs, err := sender.pushLogData(context.Background(), tt.ld)
diff --git a/exporter/splunkhecexporter/factory.go b/exporter/splunkhecexporter/factory.go
index bb828274039f..84d1c2b772d5 100644
--- a/exporter/splunkhecexporter/factory.go
+++ b/exporter/splunkhecexporter/factory.go
@@ -26,10 +26,11 @@ import (
 
 const (
 	// The value of "type" key in configuration.
-	typeStr                     = "splunk_hec"
-	defaultMaxIdleCons          = 100
-	defaultHTTPTimeout          = 10 * time.Second
-	defaultMaxContentLengthLogs = 1024 * 1024
+	typeStr                            = "splunk_hec"
+	defaultMaxIdleCons                 = 100
+	defaultHTTPTimeout                 = 10 * time.Second
+	defaultMinContentLengthCompression = 1500
+	defaultMaxContentLengthLogs        = 1024 * 1024
 )
 
 // NewFactory creates a factory for Splunk HEC exporter.
@@ -51,11 +52,12 @@ func createDefaultConfig() configmodels.Exporter {
 		TimeoutSettings: exporterhelper.TimeoutSettings{
 			Timeout: defaultHTTPTimeout,
 		},
-		RetrySettings:        exporterhelper.DefaultRetrySettings(),
-		QueueSettings:        exporterhelper.DefaultQueueSettings(),
-		DisableCompression:   false,
-		MaxConnections:       defaultMaxIdleCons,
-		MaxContentLengthLogs: defaultMaxContentLengthLogs,
+		RetrySettings:               exporterhelper.DefaultRetrySettings(),
+		QueueSettings:               exporterhelper.DefaultQueueSettings(),
+		DisableCompression:          false,
+		MaxConnections:              defaultMaxIdleCons,
+		MinContentLengthCompression: defaultMinContentLengthCompression,
+		MaxContentLengthLogs:        defaultMaxContentLengthLogs,
 	}
 }
 
diff --git a/exporter/splunkhecexporter/logdata_to_splunk.go b/exporter/splunkhecexporter/logdata_to_splunk.go
index abc5c2c36955..11baaca6149b 100644
--- a/exporter/splunkhecexporter/logdata_to_splunk.go
+++ b/exporter/splunkhecexporter/logdata_to_splunk.go
@@ -27,15 +27,15 @@ import (
 // Composite index of a log record in pdata.Logs.
 type logIndex struct {
 	// Index in orig list (i.e. root parent index).
-	origIdx int
+	resourceIdx int
 	// Index in InstrumentationLibraryLogs list (i.e. immediate parent index).
-	instIdx int
+	libraryIdx int
 	// Index in Logs list (i.e. the log record index).
-	logsIdx int
+	recordIdx int
 }
 
 func (i *logIndex) zero() bool {
-	return i.origIdx == 0 && i.instIdx == 0 && i.logsIdx == 0
+	return i.resourceIdx == 0 && i.libraryIdx == 0 && i.recordIdx == 0
 }
 
 func mapLogRecordToSplunkEvent(lr pdata.LogRecord, config *Config, logger *zap.Logger) *splunk.Event {
diff --git a/exporter/splunkhecexporter/logdata_to_splunk_test.go b/exporter/splunkhecexporter/logdata_to_splunk_test.go
index 57d305e54ba8..d3633dc2d14e 100644
--- a/exporter/splunkhecexporter/logdata_to_splunk_test.go
+++ b/exporter/splunkhecexporter/logdata_to_splunk_test.go
@@ -15,15 +15,9 @@
 package splunkhecexporter
 
 import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"math"
 	"testing"
-	"time"
 
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 	"go.opentelemetry.io/collector/consumer/pdata"
 	"go.opentelemetry.io/collector/translator/conventions"
 	"go.uber.org/zap"
@@ -31,19 +25,19 @@ import (
 	"github.com/open-telemetry/opentelemetry-collector-contrib/internal/splunk"
 )
 
-func Test_chunkEvents(t *testing.T) {
+func Test_mapLogRecordToSplunkEvent(t *testing.T) {
 	logger := zap.NewNop()
 	ts := pdata.TimestampUnixNano(123)
 
 	tests := []struct {
 		name             string
-		logDataFn        func() pdata.Logs
+		logRecordFn      func() pdata.LogRecord
 		configDataFn     func() *Config
 		wantSplunkEvents []*splunk.Event
 	}{
 		{
 			name: "valid",
-			logDataFn: func() pdata.Logs {
+			logRecordFn: func() pdata.LogRecord {
 				logRecord := pdata.NewLogRecord()
 				logRecord.Body().SetStringVal("mylog")
 				logRecord.Attributes().InsertString(conventions.AttributeServiceName, "myapp")
@@ -51,7 +45,7 @@ func Test_chunkEvents(t *testing.T) {
 				logRecord.Attributes().InsertString(conventions.AttributeHostName, "myhost")
 				logRecord.Attributes().InsertString("custom", "custom")
 				logRecord.SetTimestamp(ts)
-				return makeLog(logRecord)
+				return logRecord
 			},
 			configDataFn: func() *Config {
 				return &Config{
@@ -65,7 +59,7 @@ func Test_chunkEvents(t *testing.T) {
 		},
 		{
 			name: "non-string attribute",
-			logDataFn: func() pdata.Logs {
+			logRecordFn: func() pdata.LogRecord {
 				logRecord := pdata.NewLogRecord()
 				logRecord.Body().SetStringVal("mylog")
 				logRecord.Attributes().InsertString(conventions.AttributeServiceName, "myapp")
@@ -73,7 +67,7 @@ func Test_chunkEvents(t *testing.T) {
 				logRecord.Attributes().InsertString(conventions.AttributeHostName, "myhost")
 				logRecord.Attributes().InsertDouble("foo", 123)
 				logRecord.SetTimestamp(ts)
-				return makeLog(logRecord)
+				return logRecord
 			},
 			configDataFn: func() *Config {
 				return &Config{
@@ -87,12 +81,12 @@ func Test_chunkEvents(t *testing.T) {
 		},
 		{
 			name: "with_config",
-			logDataFn: func() pdata.Logs {
+			logRecordFn: func() pdata.LogRecord {
 				logRecord := pdata.NewLogRecord()
 				logRecord.Body().SetStringVal("mylog")
 				logRecord.Attributes().InsertString("custom", "custom")
 				logRecord.SetTimestamp(ts)
-				return makeLog(logRecord)
+				return logRecord
 			},
 			configDataFn: func() *Config {
 				return &Config{
@@ -106,9 +100,9 @@ func Test_chunkEvents(t *testing.T) {
 		},
 		{
 			name: "log_is_empty",
-			logDataFn: func() pdata.Logs {
+			logRecordFn: func() pdata.LogRecord {
 				logRecord := pdata.NewLogRecord()
-				return makeLog(logRecord)
+				return logRecord
 			},
 			configDataFn: func() *Config {
 				return &Config{
@@ -117,12 +111,12 @@ func Test_chunkEvents(t *testing.T) {
 				}
 			},
 			wantSplunkEvents: []*splunk.Event{
-				commonLogSplunkEvent(nil, 0, nil, "unknown", "source", "sourcetype"),
+				commonLogSplunkEvent(nil, 0, map[string]interface{}{}, "unknown", "source", "sourcetype"),
 			},
 		},
 		{
 			name: "with double body",
-			logDataFn: func() pdata.Logs {
+			logRecordFn: func() pdata.LogRecord {
 				logRecord := pdata.NewLogRecord()
 				logRecord.Body().SetDoubleVal(42)
 				logRecord.Attributes().InsertString(conventions.AttributeServiceName, "myapp")
@@ -130,7 +124,7 @@ func Test_chunkEvents(t *testing.T) {
 				logRecord.Attributes().InsertString(conventions.AttributeHostName, "myhost")
 				logRecord.Attributes().InsertString("custom", "custom")
 				logRecord.SetTimestamp(ts)
-				return makeLog(logRecord)
+				return logRecord
 			},
 			configDataFn: func() *Config {
 				return &Config{
@@ -144,7 +138,7 @@ func Test_chunkEvents(t *testing.T) {
 		},
 		{
 			name: "with int body",
-			logDataFn: func() pdata.Logs {
+			logRecordFn: func() pdata.LogRecord {
 				logRecord := pdata.NewLogRecord()
 				logRecord.Body().SetIntVal(42)
 				logRecord.Attributes().InsertString(conventions.AttributeServiceName, "myapp")
@@ -152,7 +146,7 @@ func Test_chunkEvents(t *testing.T) {
 				logRecord.Attributes().InsertString(conventions.AttributeHostName, "myhost")
 				logRecord.Attributes().InsertString("custom", "custom")
 				logRecord.SetTimestamp(ts)
-				return makeLog(logRecord)
+				return logRecord
 			},
 			configDataFn: func() *Config {
 				return &Config{
@@ -166,7 +160,7 @@ func Test_chunkEvents(t *testing.T) {
 		},
 		{
 			name: "with bool body",
-			logDataFn: func() pdata.Logs {
+			logRecordFn: func() pdata.LogRecord {
 				logRecord := pdata.NewLogRecord()
 				logRecord.Body().SetBoolVal(true)
 				logRecord.Attributes().InsertString(conventions.AttributeServiceName, "myapp")
@@ -174,7 +168,7 @@ func Test_chunkEvents(t *testing.T) {
 				logRecord.Attributes().InsertString(conventions.AttributeHostName, "myhost")
 				logRecord.Attributes().InsertString("custom", "custom")
 				logRecord.SetTimestamp(ts)
-				return makeLog(logRecord)
+				return logRecord
 			},
 			configDataFn: func() *Config {
 				return &Config{
@@ -188,7 +182,7 @@ func Test_chunkEvents(t *testing.T) {
 		},
 		{
 			name: "with map body",
-			logDataFn: func() pdata.Logs {
+			logRecordFn: func() pdata.LogRecord {
 				logRecord := pdata.NewLogRecord()
 				attVal := pdata.NewAttributeValueMap()
 				attMap := attVal.MapVal()
@@ -200,7 +194,7 @@ func Test_chunkEvents(t *testing.T) {
 				logRecord.Attributes().InsertString(conventions.AttributeHostName, "myhost")
 				logRecord.Attributes().InsertString("custom", "custom")
 				logRecord.SetTimestamp(ts)
-				return makeLog(logRecord)
+				return logRecord
 			},
 			configDataFn: func() *Config {
 				return &Config{
@@ -214,14 +208,14 @@ func Test_chunkEvents(t *testing.T) {
 		},
 		{
 			name: "with nil body",
-			logDataFn: func() pdata.Logs {
+			logRecordFn: func() pdata.LogRecord {
 				logRecord := pdata.NewLogRecord()
 				logRecord.Attributes().InsertString(conventions.AttributeServiceName, "myapp")
 				logRecord.Attributes().InsertString(splunk.SourcetypeLabel, "myapp-type")
 				logRecord.Attributes().InsertString(conventions.AttributeHostName, "myhost")
 				logRecord.Attributes().InsertString("custom", "custom")
 				logRecord.SetTimestamp(ts)
-				return makeLog(logRecord)
+				return logRecord
 			},
 			configDataFn: func() *Config {
 				return &Config{
@@ -235,7 +229,7 @@ func Test_chunkEvents(t *testing.T) {
 		},
 		{
 			name: "with array body",
-			logDataFn: func() pdata.Logs {
+			logRecordFn: func() pdata.LogRecord {
 				logRecord := pdata.NewLogRecord()
 				attVal := pdata.NewAttributeValueArray()
 				attArray := attVal.ArrayVal()
@@ -246,7 +240,7 @@ func Test_chunkEvents(t *testing.T) {
 				logRecord.Attributes().InsertString(conventions.AttributeHostName, "myhost")
 				logRecord.Attributes().InsertString("custom", "custom")
 				logRecord.SetTimestamp(ts)
-				return makeLog(logRecord)
+				return logRecord
 			},
 			configDataFn: func() *Config {
 				return &Config{
@@ -259,368 +253,16 @@ func Test_chunkEvents(t *testing.T) {
 			},
 		},
 	}
-
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			logs := logDataWrapper{&[]pdata.Logs{tt.logDataFn()}[0]}
-
-			ctx, cancel := context.WithCancel(context.Background())
-			defer cancel()
-
-			chunkCh := logs.chunkEvents(ctx, logger, tt.configDataFn())
-
-			events := bytes.Split(bytes.TrimSpace((<-chunkCh).buf.Bytes()), []byte("\r\n\r\n"))
-
-			require.Equal(t, len(tt.wantSplunkEvents), len(events))
-
-			var got splunk.Event
-			var gots []*splunk.Event
-
-			for i, event := range events {
-				json.Unmarshal(event, &got)
-				want := tt.wantSplunkEvents[i]
-				// float64 back to int64. int64 unmarshalled to float64 because Event is interface{}.
-				if _, ok := want.Event.(int64); ok {
-					if g, ok := got.Event.(float64); ok {
-						got.Event = int64(g)
-					}
-				}
-				assert.EqualValues(t, tt.wantSplunkEvents[i], &got)
-				gots = append(gots, &got)
+			for _, want := range tt.wantSplunkEvents {
+				got := mapLogRecordToSplunkEvent(tt.logRecordFn(), tt.configDataFn(), logger)
+				assert.EqualValues(t, want, got)
 			}
-			assert.Equal(t, tt.wantSplunkEvents, gots)
 		})
 	}
 }
 
-func Test_chunkEvents_MaxContentLength_AllEventsInChunk(t *testing.T) {
-	logs := testLogs()
-
-	_, max, events := jsonEncodeEventsBytes(logs, &Config{})
-
-	eventsLength := 0
-	for _, event := range events {
-		eventsLength += len(event)
-	}
-
-	// Chunk max content length to fit all events in 1 chunk.
-	chunkLength := len(events) * max
-	config := Config{MaxContentLengthLogs: chunkLength}
-
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	chunkCh := logs.chunkEvents(ctx, zap.NewNop(), &config)
-
-	numChunks := 0
-
-	for chunk := range chunkCh {
-		assert.NoError(t, chunk.err)
-		assert.Len(t, chunk.buf.Bytes(), eventsLength)
-		numChunks++
-	}
-
-	assert.Equal(t, 1, numChunks)
-}
-
-func Test_chunkEvents_MaxContentLength_0(t *testing.T) {
-	logs := testLogs()
-
-	_, _, events := jsonEncodeEventsBytes(logs, &Config{})
-
-	eventsLength := 0
-	for _, event := range events {
-		eventsLength += len(event)
-	}
-
-	// Chunk max content length 0 is interpreted as unlimited length.
-	chunkLength := 0
-	config := Config{MaxContentLengthLogs: chunkLength}
-
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	chunkCh := logs.chunkEvents(ctx, zap.NewNop(), &config)
-
-	numChunks := 0
-
-	for chunk := range chunkCh {
-		assert.NoError(t, chunk.err)
-		assert.Len(t, chunk.buf.Bytes(), eventsLength)
-		numChunks++
-	}
-
-	assert.Equal(t, 1, numChunks)
-}
-
-func Test_chunkEvents_MaxContentLength_0_Cancel(t *testing.T) {
-	logs := testLogs()
-
-	_, _, events := jsonEncodeEventsBytes(logs, &Config{})
-
-	eventsLength := 0
-	for _, event := range events {
-		eventsLength += len(event)
-	}
-
-	// Chunk max content length 0 is interpreted as unlimited length.
-	chunkLength := 0
-	config := Config{MaxContentLengthLogs: chunkLength}
-
-	ctx, cancel := context.WithCancel(context.Background())
-
-	chunkCh := logs.chunkEvents(ctx, zap.NewNop(), &config)
-
-	cancel()
-
-	// Giving time for chunkCh to close.
-	time.Sleep(time.Millisecond)
-
-	numChunks := 0
-
-	for chunk := range chunkCh {
-		assert.Nil(t, chunk.buf)
-		assert.NoError(t, chunk.err)
-		numChunks++
-	}
-
-	assert.Equal(t, 0, numChunks)
-}
-
-func Test_chunkEvents_MaxContentLength_Negative(t *testing.T) {
-	logs := testLogs()
-
-	_, _, events := jsonEncodeEventsBytes(logs, &Config{})
-
-	eventsLength := 0
-	for _, event := range events {
-		eventsLength += len(event)
-	}
-
-	// Negative max content length is interpreted as unlimited length.
-	chunkLength := -3
-	config := Config{MaxContentLengthLogs: chunkLength}
-
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	chunkCh := logs.chunkEvents(ctx, zap.NewNop(), &config)
-
-	numChunks := 0
-
-	for chunk := range chunkCh {
-		assert.NoError(t, chunk.err)
-		assert.Len(t, chunk.buf.Bytes(), eventsLength)
-		numChunks++
-	}
-
-	assert.Equal(t, 1, numChunks)
-}
-
-func Test_chunkEvents_MaxContentLength_Small_Error(t *testing.T) {
-	logs := testLogs()
-
-	min, _, _ := jsonEncodeEventsBytes(logs, &Config{})
-
-	// Configuring max content length to be smaller than event lengths.
-	chunkLength := min - 1
-	config := Config{MaxContentLengthLogs: chunkLength}
-
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	chunkCh := logs.chunkEvents(ctx, zap.NewNop(), &config)
-
-	numChunks := 0
-
-	for chunk := range chunkCh {
-		if chunk.err == nil {
-			numChunks++
-		}
-		assert.Nil(t, chunk.buf)
-		assert.Contains(t, chunk.err.Error(), "log event bytes exceed max content length configured")
-	}
-
-	assert.Equal(t, 0, numChunks)
-}
-
-func Test_chunkEvents_MaxContentLength_Small_Error_Cancel(t *testing.T) {
-	logs := testLogs()
-
-	min, _, _ := jsonEncodeEventsBytes(logs, &Config{})
-
-	// Configuring max content length to be smaller than event lengths.
-	chunkLength := min - 1
-	config := Config{MaxContentLengthLogs: chunkLength}
-
-	ctx, cancel := context.WithCancel(context.Background())
-
-	chunkCh := logs.chunkEvents(ctx, zap.NewNop(), &config)
-
-	cancel()
-
-	// Giving time for chunkCh to close.
-	time.Sleep(time.Millisecond)
-
-	numChunks := 0
-
-	for chunk := range chunkCh {
-		if chunk.err == nil {
-			numChunks++
-		}
-		assert.Nil(t, chunk.buf)
-		assert.NoError(t, chunk.err)
-	}
-
-	assert.Equal(t, 0, numChunks)
-}
-
-func Test_chunkEvents_MaxContentLength_1EventPerChunk(t *testing.T) {
-	logs := testLogs()
-
-	minLength, maxLength, events := jsonEncodeEventsBytes(logs, &Config{})
-
-	numEvents := len(events)
-
-	require.True(t, numEvents > 1, "More than 1 event required")
-
-	require.True(t, minLength >= maxLength/2, "Smallest event >= half largest event required")
-
-	// Setting chunk length to have 1 event per chunk.
-	chunkLength := maxLength
-	config := Config{MaxContentLengthLogs: chunkLength}
-
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	chunkCh := logs.chunkEvents(ctx, zap.NewNop(), &config)
-
-	numChunks := 0
-
-	for chunk := range chunkCh {
-		assert.NoError(t, chunk.err)
-		assert.Len(t, chunk.buf.Bytes(), len(events[numChunks]))
-		numChunks++
-	}
-
-	// Number of chunks should equal number of events.
-	assert.Equal(t, numEvents, numChunks)
-}
-
-func Test_chunkEvents_MaxContentLength_1EventPerChunk_Cancel(t *testing.T) {
-	logs := testLogs()
-
-	min, max, events := jsonEncodeEventsBytes(logs, &Config{})
-
-	require.True(t, len(events) > 1, "More than 1 event required")
-
-	require.True(t, min >= max/2, "Smallest event >= half largest event required")
-
-	// Setting chunk length to have as many chunks as there are events.
-	chunkLength := max
-	config := Config{MaxContentLengthLogs: chunkLength}
-
-	ctx, cancel := context.WithCancel(context.Background())
-
-	chunkCh := logs.chunkEvents(ctx, zap.NewNop(), &config)
-
-	_, ok := <-chunkCh
-
-	require.True(t, ok, "Chunk channel open before cancel")
-
-	cancel()
-
-	// Giving time for chunkCh to close.
-	time.Sleep(time.Millisecond)
-
-	_, ok = <-chunkCh
-
-	assert.True(t, !ok, "Chunk channel closed after cancel")
-
-}
-
-func Test_chunkEvents_MaxContentLength_2EventsPerChunk(t *testing.T) {
-	logs := testLogs()
-
-	min, max, events := jsonEncodeEventsBytes(logs, &Config{})
-
-	numEvents := len(events)
-
-	// Chunk max content length = 2 * max and this condition results in 2 event per chunk.
-	require.True(t, min >= max/2)
-
-	chunkLength := 2 * max
-	config := Config{MaxContentLengthLogs: chunkLength}
-
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	// Config max content length equal to the length of the largest event.
-	chunkCh := logs.chunkEvents(ctx, zap.NewNop(), &config)
-	defer cancel()
-
-	numChunks := 0
-
-	for chunk := range chunkCh {
-		assert.NoError(t, chunk.err)
-		numChunks++
-	}
-
-	// 2 events per chunk.
-	assert.Equal(t, numEvents, 2*numChunks)
-}
-
-func Test_chunkEvents_JSONEncodeError(t *testing.T) {
-	logs := testLogs()
-
-	// Setting a log logsIdx body to +Inf
-	logs.ResourceLogs().At(0).
-		InstrumentationLibraryLogs().At(0).
-		Logs().At(0).
-		Body().SetDoubleVal(math.Inf(1))
-
-	// JSON Encoding +Inf should trigger unsupported value error
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-
-	chunkCh := logs.chunkEvents(ctx, zap.NewNop(), &Config{})
-
-	// chunk should contain an unsupported value error triggered by JSON Encoding +Inf
-	chunk := <-chunkCh
-
-	assert.Nil(t, chunk.buf)
-	assert.Contains(t, chunk.err.Error(), "json: unsupported value: +Inf")
-
-	// the error should cause the channel to be closed.
-	_, ok := <-chunkCh
-	assert.True(t, !ok, "Events channel should be closed on error")
-}
-
-func Test_chunkEvents_JSONEncodeError_Cancel(t *testing.T) {
-	logs := testLogs()
-
-	// Setting a log logsIdx body to +Inf
-	logs.ResourceLogs().At(0).
-		InstrumentationLibraryLogs().At(0).
-		Logs().At(0).
-		Body().SetDoubleVal(math.Inf(1))
-
-	// JSON Encoding +Inf should trigger unsupported value error
-	ctx, cancel := context.WithCancel(context.Background())
-
-	chunkCh := logs.chunkEvents(ctx, zap.NewNop(), &Config{})
-
-	cancel()
-
-	// Giving time for chunkCh to close.
-	time.Sleep(time.Millisecond)
-
-	chunk, ok := <-chunkCh
-
-	assert.Nil(t, chunk)
-	assert.True(t, !ok, "Cancel should close events channel")
-}
-
 func makeLog(record pdata.LogRecord) pdata.Logs {
 	logs := pdata.NewLogs()
 	logs.ResourceLogs().Resize(1)
@@ -649,183 +291,32 @@ func commonLogSplunkEvent(
 	}
 }
 
-func Test_nilLogs(t *testing.T) {
-	logs := logDataWrapper{&[]pdata.Logs{pdata.NewLogs()}[0]}
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	chunkCh := logs.chunkEvents(ctx, zap.NewNop(), &Config{})
-	chunk := <-chunkCh
-	assert.Equal(t, 0, chunk.buf.Len())
-}
-
-func Test_nilResourceLogs(t *testing.T) {
-	logs := logDataWrapper{&[]pdata.Logs{pdata.NewLogs()}[0]}
-	logs.ResourceLogs().Resize(1)
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	chunkCh := logs.chunkEvents(ctx, zap.NewNop(), &Config{})
-	chunk := <-chunkCh
-	assert.Equal(t, 0, chunk.buf.Len())
-}
-
-func Test_nilInstrumentationLogs(t *testing.T) {
-	logs := logDataWrapper{&[]pdata.Logs{pdata.NewLogs()}[0]}
-	logs.ResourceLogs().Resize(1)
-	resourceLog := logs.ResourceLogs().At(0)
-	resourceLog.InstrumentationLibraryLogs().Resize(1)
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	chunkCh := logs.chunkEvents(ctx, zap.NewNop(), &Config{})
-	chunk := <-chunkCh
-	assert.Equal(t, 0, chunk.buf.Len())
-}
-
-func Test_nanoTimestampToEpochMilliseconds(t *testing.T) {
-	splunkTs := nanoTimestampToEpochMilliseconds(1001000000)
-	assert.Equal(t, 1.001, *splunkTs)
-	splunkTs = nanoTimestampToEpochMilliseconds(1001990000)
-	assert.Equal(t, 1.002, *splunkTs)
-	splunkTs = nanoTimestampToEpochMilliseconds(0)
-	assert.True(t, nil == splunkTs)
-}
-
-func Test_numLogs(t *testing.T) {
-	// See nested structure of logs in testLogs() comments.
-	logs := testLogs()
-
-	_0_0_0 := &logIndex{origIdx: 0, instIdx: 0, logsIdx: 0}
-	got := logs.numLogs(_0_0_0)
-
-	assert.Equal(t, 6, got)
-
-	_0_1_1 := &logIndex{origIdx: 0, instIdx: 1, logsIdx: 1}
-	got = logs.numLogs(_0_1_1)
-
-	assert.Equal(t, 4, got)
-}
-
-func Test_subLogs(t *testing.T) {
-	// See nested structure of logs in testLogs() comments.
-	logs := testLogs()
-
-	// Logs subset from leftmost index.
-	_0_0_0 := &logIndex{origIdx: 0, instIdx: 0, logsIdx: 0}
-	got := logDataWrapper{logs.subLogs(_0_0_0)}
-
-	assert.Equal(t, 6, got.numLogs(_0_0_0))
-
-	orig := *got.InternalRep().Orig
-
-	assert.Equal(t, "(0, 0, 0)", orig[0].InstrumentationLibraryLogs[0].Logs[0].Name)
-	assert.Equal(t, "(0, 1, 0)", orig[0].InstrumentationLibraryLogs[1].Logs[0].Name)
-	assert.Equal(t, "(0, 1, 1)", orig[0].InstrumentationLibraryLogs[1].Logs[1].Name)
-	assert.Equal(t, "(1, 0, 0)", orig[1].InstrumentationLibraryLogs[0].Logs[0].Name)
-	assert.Equal(t, "(1, 1, 0)", orig[1].InstrumentationLibraryLogs[1].Logs[0].Name)
-	assert.Equal(t, "(1, 2, 0)", orig[1].InstrumentationLibraryLogs[2].Logs[0].Name)
-
-	// Logs subset from rightmost index.
-	_1_2_0 := &logIndex{origIdx: 1, instIdx: 2, logsIdx: 0}
-	got = logDataWrapper{logs.subLogs(_1_2_0)}
-
-	assert.Equal(t, 1, got.numLogs(_0_0_0))
-
-	orig = *got.InternalRep().Orig
-
-	assert.Equal(t, "(1, 2, 0)", orig[0].InstrumentationLibraryLogs[0].Logs[0].Name)
-
-	// Logs subset from an in-between index.
-	_1_1_0 := &logIndex{origIdx: 1, instIdx: 1, logsIdx: 0}
-	got = logDataWrapper{logs.subLogs(_1_1_0)}
-
-	assert.Equal(t, 2, got.numLogs(_0_0_0))
-
-	orig = *got.InternalRep().Orig
-
-	assert.Equal(t, "(1, 1, 0)", orig[0].InstrumentationLibraryLogs[0].Logs[0].Name)
-	assert.Equal(t, "(1, 2, 0)", orig[0].InstrumentationLibraryLogs[1].Logs[0].Name)
-}
-
-// Creates pdata.Logs for testing.
-//
-// Structure of the pdata.Logs created showing indices:
-//
-//     0            1      <- orig index
-//    / \         / | \
-//   0   1      0  1  2    <- InstrumentationLibraryLogs index
-//  /   / \    /  /  /
-// 0   0   1  0  0  0      <- Logs index
+//func Test_nilLogs(t *testing.T) {
+//	events := logDataToSplunk(zap.NewNop(), pdata.NewLogs(), &Config{})
+//	assert.Equal(t, 0, len(events))
+//}
 //
-// The log records are named in the pattern:
-// (<orig index>, <InstrumentationLibraryLogs index>, <Logs index>)
+//func Test_nilResourceLogs(t *testing.T) {
+//	logs := pdata.NewLogs()
+//	logs.ResourceLogs().Resize(1)
+//	events := logDataToSplunk(zap.NewNop(), logs, &Config{})
+//	assert.Equal(t, 0, len(events))
+//}
 //
-// The log records are about the same size and some test depend this fact.
+//func Test_nilInstrumentationLogs(t *testing.T) {
+//	logs := pdata.NewLogs()
+//	logs.ResourceLogs().Resize(1)
+//	resourceLog := logs.ResourceLogs().At(0)
+//	resourceLog.InstrumentationLibraryLogs().Resize(1)
+//	events := logDataToSplunk(zap.NewNop(), logs, &Config{})
+//	assert.Equal(t, 0, len(events))
+//}
 //
-func testLogs() *logDataWrapper {
-	logs := logDataWrapper{&[]pdata.Logs{pdata.NewLogs()}[0]}
-	logs.ResourceLogs().Resize(2)
-
-	rl0 := logs.ResourceLogs().At(0)
-	rl0.InstrumentationLibraryLogs().Resize(2)
-
-	log := pdata.NewLogRecord()
-	log.SetName("(0, 0, 0)")
-	rl0.InstrumentationLibraryLogs().At(0).Logs().Append(log)
-
-	log = pdata.NewLogRecord()
-	log.SetName("(0, 1, 0)")
-	rl0.InstrumentationLibraryLogs().At(1).Logs().Append(log)
-
-	log = pdata.NewLogRecord()
-	log.SetName("(0, 1, 1)")
-	rl0.InstrumentationLibraryLogs().At(1).Logs().Append(log)
-
-	rl1 := logs.ResourceLogs().At(1)
-	rl1.InstrumentationLibraryLogs().Resize(3)
-
-	log = pdata.NewLogRecord()
-	log.SetName("(1, 0, 0)")
-	rl1.InstrumentationLibraryLogs().At(0).Logs().Append(log)
-
-	log = pdata.NewLogRecord()
-	log.SetName("(1, 1, 0)")
-	rl1.InstrumentationLibraryLogs().At(1).Logs().Append(log)
-
-	log = pdata.NewLogRecord()
-	log.SetName("(1, 2, 0)")
-	rl1.InstrumentationLibraryLogs().At(2).Logs().Append(log)
-
-	return &logs
-}
-
-func jsonEncodeEventsBytes(logs *logDataWrapper, config *Config) (int, int, [][]byte) {
-	events := make([][]byte, 0)
-	// min, max number of bytes of smallest, largest events.
-	var min, max int
-
-	event := new(bytes.Buffer)
-	encoder := json.NewEncoder(event)
-
-	rl := logs.ResourceLogs()
-	for i := 0; i < rl.Len(); i++ {
-		ill := rl.At(i).InstrumentationLibraryLogs()
-		for j := 0; j < ill.Len(); j++ {
-			l := ill.At(j).Logs()
-			for k := 0; k < l.Len(); k++ {
-				if err := encoder.Encode(mapLogRecordToSplunkEvent(l.At(k), config, zap.NewNop())); err == nil {
-					event.WriteString("\r\n\r\n")
-					dst := make([]byte, len(event.Bytes()))
-					copy(dst, event.Bytes())
-					events = append(events, dst)
-					if event.Len() < min || min == 0 {
-						min = event.Len()
-					}
-					if event.Len() > max {
-						max = event.Len()
-					}
-					event.Reset()
-				}
-			}
-		}
-	}
-	return min, max, events
-}
+//func Test_nanoTimestampToEpochMilliseconds(t *testing.T) {
+//	splunkTs := nanoTimestampToEpochMilliseconds(1001000000)
+//	assert.Equal(t, 1.001, *splunkTs)
+//	splunkTs = nanoTimestampToEpochMilliseconds(1001990000)
+//	assert.Equal(t, 1.002, *splunkTs)
+//	splunkTs = nanoTimestampToEpochMilliseconds(0)
+//	assert.True(t, nil == splunkTs)
+//}

From 80038fe10c0d71840a9926ebe4b88a2d8df3543a Mon Sep 17 00:00:00 2001
From: Bremer Jonathan <bjonathan@splunk.com>
Date: Mon, 22 Mar 2021 15:34:36 -0400
Subject: [PATCH 15/26] Update error message

---
 exporter/splunkhecexporter/README.md      |  3 ++-
 exporter/splunkhecexporter/client.go      | 32 ++++++++---------------
 exporter/splunkhecexporter/client_test.go |  5 ++--
 exporter/splunkhecexporter/factory.go     |  2 +-
 4 files changed, 16 insertions(+), 26 deletions(-)

diff --git a/exporter/splunkhecexporter/README.md b/exporter/splunkhecexporter/README.md
index 3638736e4197..f8841f35aa1d 100644
--- a/exporter/splunkhecexporter/README.md
+++ b/exporter/splunkhecexporter/README.md
@@ -22,7 +22,8 @@ The following configuration options can also be configured:
 - `disable_compression` (default: false): Whether to disable gzip compression over HTTP.
 - `timeout` (default: 10s): HTTP timeout when sending data.
 - `insecure_skip_verify` (default: false): Whether to skip checking the certificate of the HEC endpoint when sending data over HTTPS.
-- `max_content_length_logs` (default: 1048576): Maximum log data size in bytes per HTTP post limited to 1048576 bytes (1Mib).
+- `max_content_length_logs` (default: 1048576): Maximum log data size in bytes per HTTP post limited to 1048576 bytes (1MiB).
+- `min_content_length_compression` (default: 1500): Minimum content length in bytes to compress. 1500 is the MTU of an ethernet frame.
 
 In addition, this exporter offers queued retry which is enabled by default.
 Information about queued retry configuration parameters can be found
diff --git a/exporter/splunkhecexporter/client.go b/exporter/splunkhecexporter/client.go
index a651195a7b67..3f176cd450a3 100644
--- a/exporter/splunkhecexporter/client.go
+++ b/exporter/splunkhecexporter/client.go
@@ -177,14 +177,9 @@ func (c *client) sentLogBatch(ctx context.Context, ld pdata.Logs, send func(cont
 	var submax int
 	var index *logIndex
 	var permanentErrors []error
-	var batch *bytes.Buffer
 
 	// Provide 5000 overflow because it overruns the max content length then trims it block. Hopefully will prevent extra allocation.
-	if c.config.MaxContentLengthLogs > 0 {
-		batch = bytes.NewBuffer(make([]byte, 0, c.config.MaxContentLengthLogs+5_000))
-	} else {
-		batch = bytes.NewBuffer(make([]byte, 0))
-	}
+	batch := bytes.NewBuffer(make([]byte, 0, c.config.MaxContentLengthLogs+5_000))
 	encoder := json.NewEncoder(batch)
 
 	overflow := bytes.NewBuffer(make([]byte, 0, 5000))
@@ -201,7 +196,7 @@ func (c *client) sentLogBatch(ctx context.Context, ld pdata.Logs, send func(cont
 
 				event := mapLogRecordToSplunkEvent(logs.At(k), c.config, c.logger)
 				if err = encoder.Encode(event); err != nil {
-					permanentErrors = append(permanentErrors, consumererror.Permanent(err))
+					permanentErrors = append(permanentErrors, consumererror.Permanent(fmt.Errorf("dropped log event: %v, error: %v", event, err)))
 					continue
 				}
 				batch.WriteString("\r\n\r\n")
@@ -217,8 +212,7 @@ func (c *client) sentLogBatch(ctx context.Context, ld pdata.Logs, send func(cont
 					if over := batch.Len() - submax; over <= int(c.config.MaxContentLengthLogs) {
 						overflow.Write(batch.Bytes()[submax:batch.Len()])
 					} else {
-						err = fmt.Errorf("log event too large (configured max content length: %d bytes, event: %d bytes)", c.config.MaxContentLengthLogs, over)
-						permanentErrors = append(permanentErrors, consumererror.Permanent(err))
+						permanentErrors = append(permanentErrors, consumererror.Permanent(fmt.Errorf("dropped log event: %s, error: event size %d bytes larger than configured max content length %d bytes", string(batch.Bytes()[submax:batch.Len()]), over, c.config.MaxContentLengthLogs)))
 					}
 				}
 
@@ -254,13 +248,9 @@ func (c *client) pushLogData(ctx context.Context, ld pdata.Logs) (numDroppedLogs
 	gzipWriter := c.zippers.Get().(*gzip.Writer)
 	defer c.zippers.Put(gzipWriter)
 
-	var gzipBuf *bytes.Buffer
-	if c.config.MaxContentLengthLogs > 0 {
-		gzipBuf = bytes.NewBuffer(make([]byte, 0, c.config.MaxContentLengthLogs))
-	} else {
-		gzipBuf = bytes.NewBuffer(make([]byte, 0))
-	}
-	gzipWriter.Reset(gzipBuf)
+	gzipBuffer := bytes.NewBuffer(make([]byte, 0, c.config.MaxContentLengthLogs))
+	gzipWriter.Reset(gzipBuffer)
+
 	defer gzipWriter.Close()
 
 	// Callback when each batch is to be sent.
@@ -268,18 +258,18 @@ func (c *client) pushLogData(ctx context.Context, ld pdata.Logs) (numDroppedLogs
 		compression := buf.Len() >= int(c.config.MinContentLengthCompression) && !c.config.DisableCompression
 
 		if compression {
-			gzipBuf.Reset()
-			gzipWriter.Reset(gzipBuf)
+			gzipBuffer.Reset()
+			gzipWriter.Reset(gzipBuffer)
 
 			if _, err = io.Copy(gzipWriter, buf); err != nil {
-				return
+				return fmt.Errorf("failed copying buffer to gzip writer: %v", err)
 			}
 
 			if err = gzipWriter.Flush(); err != nil {
-				return
+				return fmt.Errorf("failed flushing compressed data to gzip writer: %v", err)
 			}
 
-			return c.postEvents(ctx, gzipBuf, compression)
+			return c.postEvents(ctx, gzipBuffer, compression)
 		}
 
 		return c.postEvents(ctx, buf, compression)
diff --git a/exporter/splunkhecexporter/client_test.go b/exporter/splunkhecexporter/client_test.go
index 386d0bd580c4..7b8f3dfccd25 100644
--- a/exporter/splunkhecexporter/client_test.go
+++ b/exporter/splunkhecexporter/client_test.go
@@ -27,9 +27,8 @@ import (
 	"testing"
 	"time"
 
-	"github.com/stretchr/testify/require"
-
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"go.opentelemetry.io/collector/component"
 	"go.opentelemetry.io/collector/component/componenttest"
 	"go.opentelemetry.io/collector/consumer/consumererror"
@@ -622,7 +621,7 @@ func Test_pushLogData_Small_MaxContentLength(t *testing.T) {
 		require.Error(t, err)
 
 		assert.True(t, consumererror.IsPermanent(err))
-		assert.Contains(t, err.Error(), "log event too large")
+		assert.Contains(t, err.Error(), "dropped log event")
 		assert.Equal(t, numLogs, numDroppedLogs)
 	}
 }
diff --git a/exporter/splunkhecexporter/factory.go b/exporter/splunkhecexporter/factory.go
index 84d1c2b772d5..853cbc3d43f4 100644
--- a/exporter/splunkhecexporter/factory.go
+++ b/exporter/splunkhecexporter/factory.go
@@ -30,7 +30,7 @@ const (
 	defaultMaxIdleCons                 = 100
 	defaultHTTPTimeout                 = 10 * time.Second
 	defaultMinContentLengthCompression = 1500
-	defaultMaxContentLengthLogs        = 1024 * 1024
+	defaultMaxContentLengthLogs        = 2 * 1024 * 1024
 )
 
 // NewFactory creates a factory for Splunk HEC exporter.

From f63e9def74d1ba572090f48072b712c46b0ae0b6 Mon Sep 17 00:00:00 2001
From: Bremer Jonathan <bjonathan@splunk.com>
Date: Tue, 23 Mar 2021 15:53:21 -0400
Subject: [PATCH 16/26] Reimplement function subLogs

---
 exporter/splunkhecexporter/client.go          | 54 ++++++++++++-----
 exporter/splunkhecexporter/client_test.go     | 58 +++++++++----------
 .../splunkhecexporter/logdata_to_splunk.go    |  8 +--
 3 files changed, 69 insertions(+), 51 deletions(-)

diff --git a/exporter/splunkhecexporter/client.go b/exporter/splunkhecexporter/client.go
index f124430efc4b..5b86c451353d 100644
--- a/exporter/splunkhecexporter/client.go
+++ b/exporter/splunkhecexporter/client.go
@@ -152,23 +152,47 @@ func (c *client) postEvents(ctx context.Context, events io.Reader, compressed bo
 	return nil
 }
 
-func subLogs(ld *pdata.Logs, logIdx *logIndex) *pdata.Logs {
-	if logIdx.zero() {
+func subLogs(ld *pdata.Logs, start *logIndex) *pdata.Logs {
+	if start.zero() {
 		return ld
 	}
-	clone := ld.Clone().InternalRep()
 
-	subset := *clone.Orig
-	subset = subset[logIdx.resourceIdx:]
-	subset[0].InstrumentationLibraryLogs = subset[0].InstrumentationLibraryLogs[logIdx.libraryIdx:]
-	subset[0].InstrumentationLibraryLogs[0].Logs = subset[0].InstrumentationLibraryLogs[0].Logs[logIdx.recordIdx:]
+	logs := pdata.NewLogs()
+	RL, RL2 := ld.ResourceLogs(), logs.ResourceLogs()
+
+	for r := start.resource; r < RL.Len(); r++ {
+		RL2.Append(pdata.NewResourceLogs())
+		RL.At(r).Resource().CopyTo(RL2.At(r - start.resource).Resource())
+
+		IL, IL2 := RL.At(r).InstrumentationLibraryLogs(), RL2.At(r-start.resource).InstrumentationLibraryLogs()
+
+		i := 0
+		if r == start.resource {
+			i = start.library
+		}
+		for i2 := 0; i < IL.Len(); i++ {
+			IL2.Append(pdata.NewInstrumentationLibraryLogs())
+			IL.At(i).InstrumentationLibrary().CopyTo(IL2.At(i2).InstrumentationLibrary())
+
+			LR, LR2 := IL.At(i).Logs(), IL2.At(i2).Logs()
+			i2++
+
+			l := 0
+			if r == start.resource && i == start.library {
+				l = start.record
+			}
+			for l2 := 0; l < LR.Len(); l++ {
+				LR2.Append(pdata.NewLogRecord())
+				LR.At(l).CopyTo(LR2.At(l2))
+				l2++
+			}
+		}
+	}
 
-	clone.Orig = &subset
-	logs := pdata.LogsFromInternalRep(clone)
 	return &logs
 }
 
-func (c *client) sentLogBatch(ctx context.Context, ld pdata.Logs, send func(context.Context, *bytes.Buffer) error) (numDroppedLogs int, err error) {
+func (c *client) sentLogBatch(ctx context.Context, ld pdata.Logs, send func(context.Context, *bytes.Buffer) error) (err error) {
 	var submax int
 	var index *logIndex
 	var permanentErrors []error
@@ -186,7 +210,7 @@ func (c *client) sentLogBatch(ctx context.Context, ld pdata.Logs, send func(cont
 			logs := ills.At(j).Logs()
 			for k := 0; k < logs.Len(); k++ {
 				if index == nil {
-					index = &logIndex{resourceIdx: i, libraryIdx: j, recordIdx: k}
+					index = &logIndex{resource: i, library: j, record: k}
 				}
 
 				event := mapLogRecordToSplunkEvent(logs.At(k), c.config, c.logger)
@@ -215,7 +239,7 @@ func (c *client) sentLogBatch(ctx context.Context, ld pdata.Logs, send func(cont
 				if batch.Len() > 0 {
 					if err = send(ctx, batch); err != nil {
 						dropped := subLogs(&ld, index)
-						return dropped.LogRecordCount(), consumererror.PartialLogsError(err, *dropped)
+						return consumererror.PartialLogsError(err, *dropped)
 					}
 				}
 				batch.Reset()
@@ -229,14 +253,14 @@ func (c *client) sentLogBatch(ctx context.Context, ld pdata.Logs, send func(cont
 	if batch.Len() > 0 {
 		if err = send(ctx, batch); err != nil {
 			dropped := subLogs(&ld, index)
-			return dropped.LogRecordCount(), consumererror.PartialLogsError(err, *dropped)
+			return consumererror.PartialLogsError(err, *dropped)
 		}
 	}
 
-	return len(permanentErrors), consumererror.CombineErrors(permanentErrors)
+	return consumererror.CombineErrors(permanentErrors)
 }
 
-func (c *client) pushLogData(ctx context.Context, ld pdata.Logs) (numDroppedLogs int, err error) {
+func (c *client) pushLogData(ctx context.Context, ld pdata.Logs) (err error) {
 	c.wg.Add(1)
 	defer c.wg.Done()
 
diff --git a/exporter/splunkhecexporter/client_test.go b/exporter/splunkhecexporter/client_test.go
index 20c5f76b00fd..d73366b32ffa 100644
--- a/exporter/splunkhecexporter/client_test.go
+++ b/exporter/splunkhecexporter/client_test.go
@@ -107,7 +107,7 @@ func createLogData(numResources int, numLibraries int, numRecords int) pdata.Log
 		for j := 0; j < numLibraries; j++ {
 			ill := rl.InstrumentationLibraryLogs().At(j)
 			for k := 0; k < numRecords; k++ {
-				ts := pdata.TimestampUnixNano(int64(k) * time.Millisecond.Nanoseconds())
+				ts := pdata.Timestamp(int64(k) * time.Millisecond.Nanoseconds())
 				logRecord := pdata.NewLogRecord()
 				logRecord.SetName(fmt.Sprintf("%d_%d_%d", i, j, k))
 				logRecord.Body().SetStringVal("mylog")
@@ -310,10 +310,10 @@ func TestReceiveLogs(t *testing.T) {
 			},
 			want: wantType{
 				batches: []string{
-					`{"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom"}}` + "\n\r\n\r\n" +
-						`{"time":0.001,"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom"}}` + "\n\r\n\r\n" +
-						`{"time":0.002,"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom"}}` + "\n\r\n\r\n" +
-						`{"time":0.003,"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom"}}` + "\n\r\n\r\n",
+					`{"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom","host.name":"myhost","service.name":"myapp"}}` + "\n\r\n\r\n" +
+						`{"time":0.001,"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom","host.name":"myhost","service.name":"myapp"}}` + "\n\r\n\r\n" +
+						`{"time":0.002,"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom","host.name":"myhost","service.name":"myapp"}}` + "\n\r\n\r\n" +
+						`{"time":0.003,"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom","host.name":"myhost","service.name":"myapp"}}` + "\n\r\n\r\n",
 				},
 				numBatches: 1,
 			},
@@ -328,15 +328,15 @@ func TestReceiveLogs(t *testing.T) {
 				cfg := NewFactory().CreateDefaultConfig().(*Config)
 				cfg.DisableCompression = disableCompression
 				cfg.MinContentLengthCompression = 4 // Given the 4 logs, 4 bytes minimum triggers compression when enable.
-				cfg.MaxContentLengthLogs = 150
+				cfg.MaxContentLengthLogs = 300
 				return cfg
 			},
 			want: wantType{
 				batches: []string{
-					`{"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom"}}` + "\n\r\n\r\n",
-					`{"time":0.001,"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom"}}` + "\n\r\n\r\n",
-					`{"time":0.002,"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom"}}` + "\n\r\n\r\n",
-					`{"time":0.003,"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom"}}` + "\n\r\n\r\n",
+					`{"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom","host.name":"myhost","service.name":"myapp"}}` + "\n\r\n\r\n",
+					`{"time":0.001,"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom","host.name":"myhost","service.name":"myapp"}}` + "\n\r\n\r\n",
+					`{"time":0.002,"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom","host.name":"myhost","service.name":"myapp"}}` + "\n\r\n\r\n",
+					`{"time":0.003,"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom","host.name":"myhost","service.name":"myapp"}}` + "\n\r\n\r\n",
 				},
 				numBatches: 4,
 			},
@@ -351,15 +351,15 @@ func TestReceiveLogs(t *testing.T) {
 				cfg := NewFactory().CreateDefaultConfig().(*Config)
 				cfg.DisableCompression = disableCompression
 				cfg.MinContentLengthCompression = 4 // Given the 4 logs, 4 bytes minimum triggers compression when enable.
-				cfg.MaxContentLengthLogs = 300
+				cfg.MaxContentLengthLogs = 400
 				return cfg
 			},
 			want: wantType{
 				batches: []string{
-					`{"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom"}}` + "\n\r\n\r\n" +
-						`{"time":0.001,"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom"}}` + "\n\r\n\r\n",
-					`{"time":0.002,"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom"}}` + "\n\r\n\r\n" +
-						`{"time":0.003,"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom"}}` + "\n\r\n\r\n",
+					`{"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom","host.name":"myhost","service.name":"myapp"}}` + "\n\r\n\r\n" +
+						`{"time":0.001,"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom","host.name":"myhost","service.name":"myapp"}}` + "\n\r\n\r\n",
+					`{"time":0.002,"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom","host.name":"myhost","service.name":"myapp"}}` + "\n\r\n\r\n" +
+						`{"time":0.003,"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom","host.name":"myhost","service.name":"myapp"}}` + "\n\r\n\r\n",
 				},
 				numBatches: 2,
 			},
@@ -568,10 +568,9 @@ func Test_pushLogData_InvalidLog(t *testing.T) {
 	log.Body().SetDoubleVal(math.Inf(1))
 	logs.ResourceLogs().At(0).InstrumentationLibraryLogs().At(0).Logs().Append(log)
 
-	numDroppedLogs, err := c.pushLogData(context.Background(), logs)
+	err := c.pushLogData(context.Background(), logs)
 
 	assert.Contains(t, err.Error(), "json: unsupported value: +Inf")
-	assert.Equal(t, 1, numDroppedLogs)
 }
 
 func Test_pushLogData_PostError(t *testing.T) {
@@ -592,10 +591,9 @@ func Test_pushLogData_PostError(t *testing.T) {
 	for _, disable := range []bool{true, false} {
 		c.config.DisableCompression = disable
 
-		numDroppedLogs, err := c.pushLogData(context.Background(), logs)
+		err := c.pushLogData(context.Background(), logs)
 		if assert.Error(t, err) {
 			assert.IsType(t, consumererror.PartialError{}, err)
-			assert.Equal(t, numLogs, numDroppedLogs)
 		}
 	}
 }
@@ -617,12 +615,11 @@ func Test_pushLogData_Small_MaxContentLength(t *testing.T) {
 	for _, disable := range []bool{true, false} {
 		c.config.DisableCompression = disable
 
-		numDroppedLogs, err := c.pushLogData(context.Background(), logs)
+		err := c.pushLogData(context.Background(), logs)
 		require.Error(t, err)
 
 		assert.True(t, consumererror.IsPermanent(err))
 		assert.Contains(t, err.Error(), "dropped log event")
-		assert.Equal(t, numLogs, numDroppedLogs)
 	}
 }
 
@@ -631,37 +628,34 @@ func TestSubLogs(t *testing.T) {
 	logs := createLogData(2, 2, 3)
 
 	// Logs subset from leftmost index (resource 0, library 0, record 0).
-	_0_0_0 := &logIndex{resourceIdx: 0, libraryIdx: 0, recordIdx: 0}
+	_0_0_0 := &logIndex{resource: 0, library: 0, record: 0}
 	got := subLogs(&logs, _0_0_0)
 
 	// Number of logs in subset should equal original logs.
 	assert.Equal(t, logs.LogRecordCount(), got.LogRecordCount())
 
-	orig := *got.InternalRep().Orig
 	// The name of the leftmost log record should be 0_0_0.
-	assert.Equal(t, "0_0_0", orig[0].InstrumentationLibraryLogs[0].Logs[0].Name)
+	assert.Equal(t, "0_0_0", got.ResourceLogs().At(0).InstrumentationLibraryLogs().At(0).Logs().At(0).Name())
 	// The name of the leftmost log record should be 1_1_2.
-	assert.Equal(t, "1_1_2", orig[1].InstrumentationLibraryLogs[1].Logs[2].Name)
+	assert.Equal(t, "1_1_2", got.ResourceLogs().At(1).InstrumentationLibraryLogs().At(1).Logs().At(2).Name())
 
 	// Logs subset from some mid index (resource 0, library 1, log 2).
-	_0_1_2 := &logIndex{resourceIdx: 0, libraryIdx: 1, recordIdx: 2}
+	_0_1_2 := &logIndex{resource: 0, library: 1, record: 2}
 	got = subLogs(&logs, _0_1_2)
 
 	assert.Equal(t, 7, got.LogRecordCount())
 
-	orig = *got.InternalRep().Orig
 	// The name of the leftmost log record should be 0_1_2.
-	assert.Equal(t, "0_1_2", orig[0].InstrumentationLibraryLogs[0].Logs[0].Name)
+	assert.Equal(t, "0_1_2", got.ResourceLogs().At(0).InstrumentationLibraryLogs().At(0).Logs().At(0).Name())
 	// The name of the rightmost log record should be 1_1_2.
-	assert.Equal(t, "1_1_2", orig[1].InstrumentationLibraryLogs[1].Logs[2].Name)
+	assert.Equal(t, "1_1_2", got.ResourceLogs().At(1).InstrumentationLibraryLogs().At(1).Logs().At(2).Name())
 
 	// Logs subset from some rightmost index (resource 0, library 1, log 2).
-	_1_1_2 := &logIndex{resourceIdx: 1, libraryIdx: 1, recordIdx: 2}
+	_1_1_2 := &logIndex{resource: 1, library: 1, record: 2}
 	got = subLogs(&logs, _1_1_2)
 
 	assert.Equal(t, 1, got.LogRecordCount())
 
-	orig = *got.InternalRep().Orig
 	// The name of the sole log record should be 1_1_2.
-	assert.Equal(t, "1_1_2", orig[0].InstrumentationLibraryLogs[0].Logs[0].Name)
+	assert.Equal(t, "1_1_2", got.ResourceLogs().At(0).InstrumentationLibraryLogs().At(0).Logs().At(0).Name())
 }
diff --git a/exporter/splunkhecexporter/logdata_to_splunk.go b/exporter/splunkhecexporter/logdata_to_splunk.go
index 1672fb7b8856..738fd74e69af 100644
--- a/exporter/splunkhecexporter/logdata_to_splunk.go
+++ b/exporter/splunkhecexporter/logdata_to_splunk.go
@@ -27,15 +27,15 @@ import (
 // Composite index of a log record in pdata.Logs.
 type logIndex struct {
 	// Index in orig list (i.e. root parent index).
-	resourceIdx int
+	resource int
 	// Index in InstrumentationLibraryLogs list (i.e. immediate parent index).
-	libraryIdx int
+	library int
 	// Index in Logs list (i.e. the log record index).
-	recordIdx int
+	record int
 }
 
 func (i *logIndex) zero() bool {
-	return i.resourceIdx == 0 && i.libraryIdx == 0 && i.recordIdx == 0
+	return i.resource == 0 && i.library == 0 && i.record == 0
 }
 
 func mapLogRecordToSplunkEvent(lr pdata.LogRecord, config *Config, logger *zap.Logger) *splunk.Event {

From abdf997d9458bb9934de3a68a4a6328a2f023907 Mon Sep 17 00:00:00 2001
From: Bremer Jonathan <bjonathan@splunk.com>
Date: Wed, 24 Mar 2021 16:15:11 -0400
Subject: [PATCH 17/26] Add unit tests

---
 exporter/splunkhecexporter/client.go          |   8 +-
 exporter/splunkhecexporter/client_test.go     | 113 ++++++++++++++++--
 exporter/splunkhecexporter/config.go          |   9 +-
 exporter/splunkhecexporter/config_test.go     |  40 ++++---
 exporter/splunkhecexporter/factory.go         |   3 +-
 .../logdata_to_splunk_test.go                 |  48 +++-----
 testbed/tests/log_test.go                     |   2 +-
 7 files changed, 157 insertions(+), 66 deletions(-)

diff --git a/exporter/splunkhecexporter/client.go b/exporter/splunkhecexporter/client.go
index 5b86c451353d..5aabc7b59690 100644
--- a/exporter/splunkhecexporter/client.go
+++ b/exporter/splunkhecexporter/client.go
@@ -153,7 +153,7 @@ func (c *client) postEvents(ctx context.Context, events io.Reader, compressed bo
 }
 
 func subLogs(ld *pdata.Logs, start *logIndex) *pdata.Logs {
-	if start.zero() {
+	if ld == nil || start == nil || start.zero() {
 		return ld
 	}
 
@@ -238,8 +238,7 @@ func (c *client) sentLogBatch(ctx context.Context, ld pdata.Logs, send func(cont
 				batch.Truncate(submax)
 				if batch.Len() > 0 {
 					if err = send(ctx, batch); err != nil {
-						dropped := subLogs(&ld, index)
-						return consumererror.PartialLogsError(err, *dropped)
+						return consumererror.PartialLogsError(err, *subLogs(&ld, index))
 					}
 				}
 				batch.Reset()
@@ -252,8 +251,7 @@ func (c *client) sentLogBatch(ctx context.Context, ld pdata.Logs, send func(cont
 
 	if batch.Len() > 0 {
 		if err = send(ctx, batch); err != nil {
-			dropped := subLogs(&ld, index)
-			return consumererror.PartialLogsError(err, *dropped)
+			return consumererror.PartialLogsError(err, *subLogs(&ld, index))
 		}
 	}
 
diff --git a/exporter/splunkhecexporter/client_test.go b/exporter/splunkhecexporter/client_test.go
index d73366b32ffa..de262b818e72 100644
--- a/exporter/splunkhecexporter/client_test.go
+++ b/exporter/splunkhecexporter/client_test.go
@@ -551,6 +551,73 @@ func TestInvalidURLClient(t *testing.T) {
 	assert.EqualError(t, err, "Permanent error: parse \"//in%20va%20lid\": invalid URL escape \"%20\"")
 }
 
+func Test_pushLogData_nil_Logs(t *testing.T) {
+	tests := []struct {
+		name     func(bool) string
+		logs     pdata.Logs
+		requires func(*testing.T, pdata.Logs)
+	}{
+		{
+			name: func(disable bool) string {
+				return "COMPRESSION " + map[bool]string{true: "DISABLED ", false: "ENABLED "}[disable] + "nil ResourceLogs"
+			},
+			logs: pdata.NewLogs(),
+			requires: func(t *testing.T, logs pdata.Logs) {
+				require.Zero(t, logs.ResourceLogs().Len())
+			},
+		},
+		{
+			name: func(disable bool) string {
+				return "COMPRESSION " + map[bool]string{true: "DISABLED ", false: "ENABLED "}[disable] + "nil InstrumentationLogs"
+			},
+			logs: func() pdata.Logs {
+				logs := pdata.NewLogs()
+				logs.ResourceLogs().Resize(1)
+				logs.ResourceLogs().At(0).InstrumentationLibraryLogs()
+				return logs
+			}(),
+			requires: func(t *testing.T, logs pdata.Logs) {
+				require.Equal(t, logs.ResourceLogs().Len(), 1)
+				require.Zero(t, logs.ResourceLogs().At(0).InstrumentationLibraryLogs().Len())
+			},
+		},
+		{
+			name: func(disable bool) string {
+				return "COMPRESSION " + map[bool]string{true: "DISABLED ", false: "ENABLED "}[disable] + "nil LogRecords"
+			},
+			logs: func() pdata.Logs {
+				logs := pdata.NewLogs()
+				logs.ResourceLogs().Resize(1)
+				logs.ResourceLogs().At(0).InstrumentationLibraryLogs().Resize(1)
+				return logs
+			}(),
+			requires: func(t *testing.T, logs pdata.Logs) {
+				require.Equal(t, logs.ResourceLogs().Len(), 1)
+				require.Equal(t, logs.ResourceLogs().At(0).InstrumentationLibraryLogs().Len(), 1)
+				require.Zero(t, logs.ResourceLogs().At(0).InstrumentationLibraryLogs().At(0).Logs().Len())
+			},
+		},
+	}
+
+	c := client{
+		zippers: sync.Pool{New: func() interface{} {
+			return gzip.NewWriter(nil)
+		}},
+		config: NewFactory().CreateDefaultConfig().(*Config),
+	}
+
+	for _, test := range tests {
+		for _, disabled := range []bool{true, false} {
+			t.Run(test.name(disabled), func(t *testing.T) {
+				test.requires(t, test.logs)
+				err := c.pushLogData(context.Background(), test.logs)
+				assert.NoError(t, err)
+			})
+		}
+	}
+
+}
+
 func Test_pushLogData_InvalidLog(t *testing.T) {
 	c := client{
 		zippers: sync.Pool{New: func() interface{} {
@@ -582,20 +649,40 @@ func Test_pushLogData_PostError(t *testing.T) {
 		config: NewFactory().CreateDefaultConfig().(*Config),
 	}
 
-	numLogs := 1500
-	logs := createLogData(1, 1, numLogs)
-
-	// Given 1500 logs, 1024 bytes is small enough to trigger compression when compression enable.
-	c.config.MinContentLengthCompression = 1024
-
-	for _, disable := range []bool{true, false} {
-		c.config.DisableCompression = disable
+	// 1500 log records -> ~371888 bytes when JSON encoded.
+	logs := createLogData(1, 1, 2000)
 
-		err := c.pushLogData(context.Background(), logs)
-		if assert.Error(t, err) {
-			assert.IsType(t, consumererror.PartialError{}, err)
-		}
-	}
+	// 0 -> unlimited size batch, true -> compression disabled.
+	c.config.MaxContentLengthLogs, c.config.DisableCompression = 0, true
+	err := c.pushLogData(context.Background(), logs)
+	require.Error(t, err)
+	assert.IsType(t, consumererror.PartialError{}, err)
+	assert.Equal(t, (err.(consumererror.PartialError)).GetLogs(), logs)
+
+	// 0 -> unlimited size batch, true -> compression enabled.
+	c.config.MaxContentLengthLogs, c.config.DisableCompression = 0, false
+	// 1500 < 371888 -> compression occurs.
+	c.config.MinContentLengthCompression = 1500
+	err = c.pushLogData(context.Background(), logs)
+	require.Error(t, err)
+	assert.IsType(t, consumererror.PartialError{}, err)
+	assert.Equal(t, (err.(consumererror.PartialError)).GetLogs(), logs)
+
+	// 200000 < 371888 -> multiple batches, true -> compression disabled.
+	c.config.MaxContentLengthLogs, c.config.DisableCompression = 200_000, true
+	err = c.pushLogData(context.Background(), logs)
+	require.Error(t, err)
+	assert.IsType(t, consumererror.PartialError{}, err)
+	assert.Equal(t, (err.(consumererror.PartialError)).GetLogs(), logs)
+
+	// 200000 < 371888 -> multiple batches, true -> compression enabled.
+	c.config.MaxContentLengthLogs, c.config.DisableCompression = 200_000, false
+	// 1500 < 200000 -> compression occurs.
+	c.config.MinContentLengthCompression = 1500
+	err = c.pushLogData(context.Background(), logs)
+	require.Error(t, err)
+	assert.IsType(t, consumererror.PartialError{}, err)
+	assert.Equal(t, (err.(consumererror.PartialError)).GetLogs(), logs)
 }
 
 func Test_pushLogData_Small_MaxContentLength(t *testing.T) {
diff --git a/exporter/splunkhecexporter/config.go b/exporter/splunkhecexporter/config.go
index a2f7226a9503..108a09b1ffc5 100644
--- a/exporter/splunkhecexporter/config.go
+++ b/exporter/splunkhecexporter/config.go
@@ -26,7 +26,8 @@ import (
 
 const (
 	// hecPath is the default HEC path on the Splunk instance.
-	hecPath = "services/collector"
+	hecPath                   = "services/collector"
+	maxContentLengthLogsLimit = 2 * 1024 * 1024
 )
 
 // Config defines configuration for Splunk exporter.
@@ -64,7 +65,7 @@ type Config struct {
 	// insecure_skip_verify skips checking the certificate of the HEC endpoint when sending data over HTTPS. Defaults to false.
 	InsecureSkipVerify bool `mapstructure:"insecure_skip_verify"`
 
-	// MaxContentLengthLogs is the maximum log data size in bytes per HTTP post. Defaults to the backend limit of 1048576 bytes (1MiB).
+	// Maximum log data size in bytes per HTTP post. Defaults to the backend limit of 2097152 bytes (2MiB).
 	MaxContentLengthLogs uint `mapstructure:"max_content_length_logs"`
 }
 
@@ -93,6 +94,10 @@ func (cfg *Config) validateConfig() error {
 		return errors.New(`requires a non-empty "token"`)
 	}
 
+	if cfg.MaxContentLengthLogs > maxContentLengthLogsLimit {
+		return fmt.Errorf(`requires "max_content_length_logs" <= %d`, maxContentLengthLogsLimit)
+	}
+
 	return nil
 }
 
diff --git a/exporter/splunkhecexporter/config_test.go b/exporter/splunkhecexporter/config_test.go
index 775aef3bca89..452c713be708 100644
--- a/exporter/splunkhecexporter/config_test.go
+++ b/exporter/splunkhecexporter/config_test.go
@@ -64,8 +64,8 @@ func TestLoadConfig(t *testing.T) {
 		SourceType:                  "otel",
 		Index:                       "metrics",
 		MaxConnections:              100,
-		MinContentLengthCompression: defaultMinContentLengthCompression,
-		MaxContentLengthLogs:        defaultMaxContentLengthLogs,
+		MinContentLengthCompression: 1500,
+		MaxContentLengthLogs:        2 * 1024 * 1024,
 		TimeoutSettings: exporterhelper.TimeoutSettings{
 			Timeout: 10 * time.Second,
 		},
@@ -91,12 +91,13 @@ func TestLoadConfig(t *testing.T) {
 
 func TestConfig_getOptionsFromConfig(t *testing.T) {
 	type fields struct {
-		ExporterSettings configmodels.ExporterSettings
-		Endpoint         string
-		Token            string
-		Source           string
-		SourceType       string
-		Index            string
+		ExporterSettings     configmodels.ExporterSettings
+		Endpoint             string
+		Token                string
+		Source               string
+		SourceType           string
+		Index                string
+		MaxContentLengthLogs uint
 	}
 	tests := []struct {
 		name    string
@@ -141,16 +142,27 @@ func TestConfig_getOptionsFromConfig(t *testing.T) {
 			want:    nil,
 			wantErr: true,
 		},
+		{
+			name: "Test max content length logs greater than limit",
+			fields: fields{
+				Token:                "1234",
+				Endpoint:             "https://example.com:8000",
+				MaxContentLengthLogs: maxContentLengthLogsLimit + 1,
+			},
+			want:    nil,
+			wantErr: true,
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			cfg := &Config{
-				ExporterSettings: tt.fields.ExporterSettings,
-				Token:            tt.fields.Token,
-				Endpoint:         tt.fields.Endpoint,
-				Source:           tt.fields.Source,
-				SourceType:       tt.fields.SourceType,
-				Index:            tt.fields.Index,
+				ExporterSettings:     tt.fields.ExporterSettings,
+				Token:                tt.fields.Token,
+				Endpoint:             tt.fields.Endpoint,
+				Source:               tt.fields.Source,
+				SourceType:           tt.fields.SourceType,
+				Index:                tt.fields.Index,
+				MaxContentLengthLogs: tt.fields.MaxContentLengthLogs,
 			}
 			got, err := cfg.getOptionsFromConfig()
 			if (err != nil) != tt.wantErr {
diff --git a/exporter/splunkhecexporter/factory.go b/exporter/splunkhecexporter/factory.go
index 853cbc3d43f4..5759556b54c8 100644
--- a/exporter/splunkhecexporter/factory.go
+++ b/exporter/splunkhecexporter/factory.go
@@ -30,7 +30,6 @@ const (
 	defaultMaxIdleCons                 = 100
 	defaultHTTPTimeout                 = 10 * time.Second
 	defaultMinContentLengthCompression = 1500
-	defaultMaxContentLengthLogs        = 2 * 1024 * 1024
 )
 
 // NewFactory creates a factory for Splunk HEC exporter.
@@ -57,7 +56,7 @@ func createDefaultConfig() configmodels.Exporter {
 		DisableCompression:          false,
 		MaxConnections:              defaultMaxIdleCons,
 		MinContentLengthCompression: defaultMinContentLengthCompression,
-		MaxContentLengthLogs:        defaultMaxContentLengthLogs,
+		MaxContentLengthLogs:        maxContentLengthLogsLimit,
 	}
 }
 
diff --git a/exporter/splunkhecexporter/logdata_to_splunk_test.go b/exporter/splunkhecexporter/logdata_to_splunk_test.go
index 3c8fadbfe4fa..640cf23ee69d 100644
--- a/exporter/splunkhecexporter/logdata_to_splunk_test.go
+++ b/exporter/splunkhecexporter/logdata_to_splunk_test.go
@@ -296,32 +296,22 @@ func commonLogSplunkEvent(
 	}
 }
 
-//func Test_nilLogs(t *testing.T) {
-//	events := logDataToSplunk(zap.NewNop(), pdata.NewLogs(), &Config{})
-//	assert.Equal(t, 0, len(events))
-//}
-//
-//func Test_nilResourceLogs(t *testing.T) {
-//	logs := pdata.NewLogs()
-//	logs.ResourceLogs().Resize(1)
-//	events := logDataToSplunk(zap.NewNop(), logs, &Config{})
-//	assert.Equal(t, 0, len(events))
-//}
-//
-//func Test_nilInstrumentationLogs(t *testing.T) {
-//	logs := pdata.NewLogs()
-//	logs.ResourceLogs().Resize(1)
-//	resourceLog := logs.ResourceLogs().At(0)
-//	resourceLog.InstrumentationLibraryLogs().Resize(1)
-//	events := logDataToSplunk(zap.NewNop(), logs, &Config{})
-//	assert.Equal(t, 0, len(events))
-//}
-//
-//func Test_nanoTimestampToEpochMilliseconds(t *testing.T) {
-//	splunkTs := nanoTimestampToEpochMilliseconds(1001000000)
-//	assert.Equal(t, 1.001, *splunkTs)
-//	splunkTs = nanoTimestampToEpochMilliseconds(1001990000)
-//	assert.Equal(t, 1.002, *splunkTs)
-//	splunkTs = nanoTimestampToEpochMilliseconds(0)
-//	assert.True(t, nil == splunkTs)
-//}
+func Test_emptyLogRecord(t *testing.T) {
+	event := mapLogRecordToSplunkEvent(pdata.NewLogRecord(), &Config{}, zap.NewNop())
+	assert.Nil(t, event.Time)
+	assert.Equal(t, event.Host, "unknown")
+	assert.Zero(t, event.Source)
+	assert.Zero(t, event.SourceType)
+	assert.Zero(t, event.Index)
+	assert.Nil(t, event.Event)
+	assert.Empty(t, event.Fields)
+}
+
+func Test_nanoTimestampToEpochMilliseconds(t *testing.T) {
+	splunkTs := nanoTimestampToEpochMilliseconds(1001000000)
+	assert.Equal(t, 1.001, *splunkTs)
+	splunkTs = nanoTimestampToEpochMilliseconds(1001990000)
+	assert.Equal(t, 1.002, *splunkTs)
+	splunkTs = nanoTimestampToEpochMilliseconds(0)
+	assert.True(t, nil == splunkTs)
+}
diff --git a/testbed/tests/log_test.go b/testbed/tests/log_test.go
index 922aafadc8c5..e25f1e7c70b2 100644
--- a/testbed/tests/log_test.go
+++ b/testbed/tests/log_test.go
@@ -39,7 +39,7 @@ func TestLog10kDPS(t *testing.T) {
 			sender:   testbed.NewOTLPLogsDataSender(testbed.DefaultHost, testbed.GetAvailablePort(t)),
 			receiver: testbed.NewOTLPDataReceiver(testbed.GetAvailablePort(t)),
 			resourceSpec: testbed.ResourceSpec{
-				ExpectedMaxCPU: 26,
+				ExpectedMaxCPU: 30,
 				ExpectedMaxRAM: 82,
 			},
 		},

From 3d31393123725652d2f6162c8ebf77ac39a96124 Mon Sep 17 00:00:00 2001
From: Bremer Jonathan <bjonathan@splunk.com>
Date: Wed, 24 Mar 2021 16:19:18 -0400
Subject: [PATCH 18/26] Update readme

---
 exporter/splunkhecexporter/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/exporter/splunkhecexporter/README.md b/exporter/splunkhecexporter/README.md
index f8841f35aa1d..df3279e3f9fd 100644
--- a/exporter/splunkhecexporter/README.md
+++ b/exporter/splunkhecexporter/README.md
@@ -22,7 +22,7 @@ The following configuration options can also be configured:
 - `disable_compression` (default: false): Whether to disable gzip compression over HTTP.
 - `timeout` (default: 10s): HTTP timeout when sending data.
 - `insecure_skip_verify` (default: false): Whether to skip checking the certificate of the HEC endpoint when sending data over HTTPS.
-- `max_content_length_logs` (default: 1048576): Maximum log data size in bytes per HTTP post limited to 1048576 bytes (1MiB).
+- `max_content_length_logs` (default: 2097152): Maximum log data size in bytes per HTTP post limited to 2097152 bytes (2 MiB).
 - `min_content_length_compression` (default: 1500): Minimum content length in bytes to compress. 1500 is the MTU of an ethernet frame.
 
 In addition, this exporter offers queued retry which is enabled by default.

From 72f60e9b7f99f50de729fb180fc89ae8959eb45c Mon Sep 17 00:00:00 2001
From: Bremer Jonathan <bjonathan@splunk.com>
Date: Wed, 24 Mar 2021 17:19:20 -0400
Subject: [PATCH 19/26] Fix comment

---
 exporter/splunkhecexporter/client_test.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/exporter/splunkhecexporter/client_test.go b/exporter/splunkhecexporter/client_test.go
index de262b818e72..1cbb3dc8b95d 100644
--- a/exporter/splunkhecexporter/client_test.go
+++ b/exporter/splunkhecexporter/client_test.go
@@ -649,7 +649,7 @@ func Test_pushLogData_PostError(t *testing.T) {
 		config: NewFactory().CreateDefaultConfig().(*Config),
 	}
 
-	// 1500 log records -> ~371888 bytes when JSON encoded.
+	// 2000 log records -> ~371888 bytes when JSON encoded.
 	logs := createLogData(1, 1, 2000)
 
 	// 0 -> unlimited size batch, true -> compression disabled.
@@ -669,14 +669,14 @@ func Test_pushLogData_PostError(t *testing.T) {
 	assert.Equal(t, (err.(consumererror.PartialError)).GetLogs(), logs)
 
 	// 200000 < 371888 -> multiple batches, true -> compression disabled.
-	c.config.MaxContentLengthLogs, c.config.DisableCompression = 200_000, true
+	c.config.MaxContentLengthLogs, c.config.DisableCompression = 200000, true
 	err = c.pushLogData(context.Background(), logs)
 	require.Error(t, err)
 	assert.IsType(t, consumererror.PartialError{}, err)
 	assert.Equal(t, (err.(consumererror.PartialError)).GetLogs(), logs)
 
-	// 200000 < 371888 -> multiple batches, true -> compression enabled.
-	c.config.MaxContentLengthLogs, c.config.DisableCompression = 200_000, false
+	// 200000 < 371888 -> multiple batches, false -> compression enabled.
+	c.config.MaxContentLengthLogs, c.config.DisableCompression = 200000, false
 	// 1500 < 200000 -> compression occurs.
 	c.config.MinContentLengthCompression = 1500
 	err = c.pushLogData(context.Background(), logs)

From d92b6d79913bb86f41dd37d688a78f05d7c6218e Mon Sep 17 00:00:00 2001
From: Bremer Jonathan <bjonathan@splunk.com>
Date: Thu, 25 Mar 2021 15:44:21 -0400
Subject: [PATCH 20/26] Rename local variable

Co-authored-by: Tigran Najaryan <4194920+tigrannajaryan@users.noreply.github.com>
---
 exporter/splunkhecexporter/client.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/exporter/splunkhecexporter/client.go b/exporter/splunkhecexporter/client.go
index 5aabc7b59690..c266f14390e3 100644
--- a/exporter/splunkhecexporter/client.go
+++ b/exporter/splunkhecexporter/client.go
@@ -272,7 +272,7 @@ func (c *client) pushLogData(ctx context.Context, ld pdata.Logs) (err error) {
 
 	// Callback when each batch is to be sent.
 	send := func(ctx context.Context, buf *bytes.Buffer) (err error) {
-		compression := buf.Len() >= int(c.config.MinContentLengthCompression) && !c.config.DisableCompression
+		shouldCompress := buf.Len() >= int(c.config.MinContentLengthCompression) && !c.config.DisableCompression
 
 		if compression {
 			gzipBuffer.Reset()

From 13ab6a35f124791dd3bea8dd2585baeb5b1f8e1f Mon Sep 17 00:00:00 2001
From: Bremer Jonathan <bjonathan@splunk.com>
Date: Fri, 26 Mar 2021 15:04:46 -0400
Subject: [PATCH 21/26] Refactor and add comments

---
 exporter/splunkhecexporter/client.go      | 230 +++++++++++-----------
 exporter/splunkhecexporter/client_test.go |   7 +-
 2 files changed, 122 insertions(+), 115 deletions(-)

diff --git a/exporter/splunkhecexporter/client.go b/exporter/splunkhecexporter/client.go
index c266f14390e3..f6572a45543d 100644
--- a/exporter/splunkhecexporter/client.go
+++ b/exporter/splunkhecexporter/client.go
@@ -119,98 +119,67 @@ func (c *client) sendSplunkEvents(ctx context.Context, splunkEvents []*splunk.Ev
 	return c.postEvents(ctx, body, compressed)
 }
 
-func (c *client) postEvents(ctx context.Context, events io.Reader, compressed bool) error {
-	req, err := http.NewRequestWithContext(ctx, "POST", c.url.String(), events)
-	if err != nil {
-		return consumererror.Permanent(err)
-	}
-
-	for k, v := range c.headers {
-		req.Header.Set(k, v)
-	}
-
-	if compressed {
-		req.Header.Set("Content-Encoding", "gzip")
-	}
+func (c *client) pushLogData(ctx context.Context, ld pdata.Logs) (err error) {
+	c.wg.Add(1)
+	defer c.wg.Done()
 
-	resp, err := c.client.Do(req)
-	if err != nil {
-		return err
-	}
+	gzipWriter := c.zippers.Get().(*gzip.Writer)
+	defer c.zippers.Put(gzipWriter)
 
-	io.Copy(ioutil.Discard, resp.Body)
-	resp.Body.Close()
+	gzipBuffer := bytes.NewBuffer(make([]byte, 0, c.config.MaxContentLengthLogs))
+	gzipWriter.Reset(gzipBuffer)
 
-	// Splunk accepts all 2XX codes.
-	if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
-		err = fmt.Errorf(
-			"HTTP %d %q",
-			resp.StatusCode,
-			http.StatusText(resp.StatusCode))
-		return err
-	}
-	return nil
-}
+	defer gzipWriter.Close()
 
-func subLogs(ld *pdata.Logs, start *logIndex) *pdata.Logs {
-	if ld == nil || start == nil || start.zero() {
-		return ld
-	}
+	// Callback when each batch is to be sent.
+	send := func(ctx context.Context, buf *bytes.Buffer) (err error) {
+		shouldCompress := buf.Len() >= int(c.config.MinContentLengthCompression) && !c.config.DisableCompression
 
-	logs := pdata.NewLogs()
-	RL, RL2 := ld.ResourceLogs(), logs.ResourceLogs()
+		if shouldCompress {
+			gzipBuffer.Reset()
+			gzipWriter.Reset(gzipBuffer)
 
-	for r := start.resource; r < RL.Len(); r++ {
-		RL2.Append(pdata.NewResourceLogs())
-		RL.At(r).Resource().CopyTo(RL2.At(r - start.resource).Resource())
+			if _, err = io.Copy(gzipWriter, buf); err != nil {
+				return fmt.Errorf("failed copying buffer to gzip writer: %v", err)
+			}
 
-		IL, IL2 := RL.At(r).InstrumentationLibraryLogs(), RL2.At(r-start.resource).InstrumentationLibraryLogs()
+			if err = gzipWriter.Flush(); err != nil {
+				return fmt.Errorf("failed flushing compressed data to gzip writer: %v", err)
+			}
 
-		i := 0
-		if r == start.resource {
-			i = start.library
+			return c.postEvents(ctx, gzipBuffer, shouldCompress)
 		}
-		for i2 := 0; i < IL.Len(); i++ {
-			IL2.Append(pdata.NewInstrumentationLibraryLogs())
-			IL.At(i).InstrumentationLibrary().CopyTo(IL2.At(i2).InstrumentationLibrary())
 
-			LR, LR2 := IL.At(i).Logs(), IL2.At(i2).Logs()
-			i2++
-
-			l := 0
-			if r == start.resource && i == start.library {
-				l = start.record
-			}
-			for l2 := 0; l < LR.Len(); l++ {
-				LR2.Append(pdata.NewLogRecord())
-				LR.At(l).CopyTo(LR2.At(l2))
-				l2++
-			}
-		}
+		return c.postEvents(ctx, buf, shouldCompress)
 	}
 
-	return &logs
+	return c.pushLogDataParts(ctx, ld, send)
 }
 
-func (c *client) sentLogBatch(ctx context.Context, ld pdata.Logs, send func(context.Context, *bytes.Buffer) error) (err error) {
-	var submax int
-	var index *logIndex
+// pushLogDataParts partitions log data then pushes the parts.
+func (c *client) pushLogDataParts(ctx context.Context, ld pdata.Logs, send func(context.Context, *bytes.Buffer) error) (err error) {
+	// max is the maximum number of bytes allowed in logs buffer.
+	// pad4096 pads max by 4096 to prevent reallocation in logs buffer prior to truncation. It is assumed that a single log is likely less than 4096 bytes.
+	var max, pad4096 = c.config.MaxContentLengthLogs, uint(4096)
+	// Buffer of a part of log data.
+	var logsBuf = bytes.NewBuffer(make([]byte, 0, max+pad4096))
+	var encoder = json.NewEncoder(logsBuf)
+	// Number of bytes below max of logs in logsBuf.
+	var length int
+	// Temp storage for log over max in logsBuf.
+	var overMaxBuf = bytes.NewBuffer(make([]byte, 0, pad4096))
+	// Index of the first log in a logs part.
+	var log0Index *logIndex
 	var permanentErrors []error
+	var rls = ld.ResourceLogs()
 
-	// Provide 5000 overflow because it overruns the max content length then trims it block. Hopefully will prevent extra allocation.
-	batch := bytes.NewBuffer(make([]byte, 0, c.config.MaxContentLengthLogs+5_000))
-	encoder := json.NewEncoder(batch)
-
-	overflow := bytes.NewBuffer(make([]byte, 0, 5000))
-
-	rls := ld.ResourceLogs()
 	for i := 0; i < rls.Len(); i++ {
 		ills := rls.At(i).InstrumentationLibraryLogs()
 		for j := 0; j < ills.Len(); j++ {
 			logs := ills.At(j).Logs()
 			for k := 0; k < logs.Len(); k++ {
-				if index == nil {
-					index = &logIndex{resource: i, library: j, record: k}
+				if log0Index == nil {
+					log0Index = &logIndex{resource: i, library: j, record: k}
 				}
 
 				event := mapLogRecordToSplunkEvent(logs.At(k), c.config, c.logger)
@@ -218,81 +187,118 @@ func (c *client) sentLogBatch(ctx context.Context, ld pdata.Logs, send func(cont
 					permanentErrors = append(permanentErrors, consumererror.Permanent(fmt.Errorf("dropped log event: %v, error: %v", event, err)))
 					continue
 				}
-				batch.WriteString("\r\n\r\n")
+				logsBuf.WriteString("\r\n\r\n")
 
 				// Consistent with ContentLength in http.Request, MaxContentLengthLogs value of 0 indicates length unknown (i.e. unbound).
-				if c.config.MaxContentLengthLogs == 0 || batch.Len() <= int(c.config.MaxContentLengthLogs) {
-					submax = batch.Len()
+				if max == 0 || logsBuf.Len() <= int(max) {
+					length = logsBuf.Len()
 					continue
 				}
 
-				overflow.Reset()
-				if c.config.MaxContentLengthLogs > 0 {
-					if over := batch.Len() - submax; over <= int(c.config.MaxContentLengthLogs) {
-						overflow.Write(batch.Bytes()[submax:batch.Len()])
+				overMaxBuf.Reset()
+				if max > 0 {
+					if overLength := logsBuf.Len() - length; overLength <= int(max) {
+						overMaxBuf.Write(logsBuf.Bytes()[length:logsBuf.Len()])
 					} else {
-						permanentErrors = append(permanentErrors, consumererror.Permanent(fmt.Errorf("dropped log event: %s, error: event size %d bytes larger than configured max content length %d bytes", string(batch.Bytes()[submax:batch.Len()]), over, c.config.MaxContentLengthLogs)))
+						permanentErrors = append(permanentErrors, consumererror.Permanent(fmt.Errorf("dropped log event: %s, error: event size %d bytes larger than configured max content length %d bytes", string(logsBuf.Bytes()[length:logsBuf.Len()]), overLength, max)))
 					}
 				}
 
-				batch.Truncate(submax)
-				if batch.Len() > 0 {
-					if err = send(ctx, batch); err != nil {
-						return consumererror.PartialLogsError(err, *subLogs(&ld, index))
+				logsBuf.Truncate(length)
+				if logsBuf.Len() > 0 {
+					if err = send(ctx, logsBuf); err != nil {
+						return consumererror.PartialLogsError(err, *subLogs(&ld, log0Index))
 					}
 				}
-				batch.Reset()
-				overflow.WriteTo(batch)
+				logsBuf.Reset()
+				overMaxBuf.WriteTo(logsBuf)
 
-				index, submax = nil, batch.Len()
+				log0Index, length = nil, logsBuf.Len()
 			}
 		}
 	}
 
-	if batch.Len() > 0 {
-		if err = send(ctx, batch); err != nil {
-			return consumererror.PartialLogsError(err, *subLogs(&ld, index))
+	if logsBuf.Len() > 0 {
+		if err = send(ctx, logsBuf); err != nil {
+			return consumererror.PartialLogsError(err, *subLogs(&ld, log0Index))
 		}
 	}
 
 	return consumererror.CombineErrors(permanentErrors)
 }
 
-func (c *client) pushLogData(ctx context.Context, ld pdata.Logs) (err error) {
-	c.wg.Add(1)
-	defer c.wg.Done()
+func (c *client) postEvents(ctx context.Context, events io.Reader, compressed bool) error {
+	req, err := http.NewRequestWithContext(ctx, "POST", c.url.String(), events)
+	if err != nil {
+		return consumererror.Permanent(err)
+	}
 
-	gzipWriter := c.zippers.Get().(*gzip.Writer)
-	defer c.zippers.Put(gzipWriter)
+	for k, v := range c.headers {
+		req.Header.Set(k, v)
+	}
 
-	gzipBuffer := bytes.NewBuffer(make([]byte, 0, c.config.MaxContentLengthLogs))
-	gzipWriter.Reset(gzipBuffer)
+	if compressed {
+		req.Header.Set("Content-Encoding", "gzip")
+	}
 
-	defer gzipWriter.Close()
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return err
+	}
 
-	// Callback when each batch is to be sent.
-	send := func(ctx context.Context, buf *bytes.Buffer) (err error) {
-		shouldCompress := buf.Len() >= int(c.config.MinContentLengthCompression) && !c.config.DisableCompression
+	io.Copy(ioutil.Discard, resp.Body)
+	resp.Body.Close()
 
-		if compression {
-			gzipBuffer.Reset()
-			gzipWriter.Reset(gzipBuffer)
+	// Splunk accepts all 2XX codes.
+	if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
+		err = fmt.Errorf(
+			"HTTP %d %q",
+			resp.StatusCode,
+			http.StatusText(resp.StatusCode))
+		return err
+	}
+	return nil
+}
 
-			if _, err = io.Copy(gzipWriter, buf); err != nil {
-				return fmt.Errorf("failed copying buffer to gzip writer: %v", err)
-			}
+func subLogs(ld *pdata.Logs, start *logIndex) *pdata.Logs {
+	if ld == nil || start == nil || start.zero() {
+		return ld
+	}
 
-			if err = gzipWriter.Flush(); err != nil {
-				return fmt.Errorf("failed flushing compressed data to gzip writer: %v", err)
-			}
+	logs := pdata.NewLogs()
 
-			return c.postEvents(ctx, gzipBuffer, compression)
+	rl, rl2 := ld.ResourceLogs(), logs.ResourceLogs()
+
+	for r := start.resource; r < rl.Len(); r++ {
+		rl2.Append(pdata.NewResourceLogs())
+		rl.At(r).Resource().CopyTo(rl2.At(r - start.resource).Resource())
+
+		il, il2 := rl.At(r).InstrumentationLibraryLogs(), rl2.At(r-start.resource).InstrumentationLibraryLogs()
+
+		i := 0
+		if r == start.resource {
+			i = start.library
 		}
+		for i2 := 0; i < il.Len(); i++ {
+			il2.Append(pdata.NewInstrumentationLibraryLogs())
+			il.At(i).InstrumentationLibrary().CopyTo(il2.At(i2).InstrumentationLibrary())
+
+			lr, lr2 := il.At(i).Logs(), il2.At(i2).Logs()
+			i2++
 
-		return c.postEvents(ctx, buf, compression)
+			l := 0
+			if r == start.resource && i == start.library {
+				l = start.record
+			}
+			for l2 := 0; l < lr.Len(); l++ {
+				lr2.Append(pdata.NewLogRecord())
+				lr.At(l).CopyTo(lr2.At(l2))
+				l2++
+			}
+		}
 	}
 
-	return c.sentLogBatch(ctx, ld, send)
+	return &logs
 }
 
 func encodeBodyEvents(zippers *sync.Pool, evs []*splunk.Event, disableCompression bool, minContentLengthCompression uint) (bodyReader io.Reader, compressed bool, err error) {
diff --git a/exporter/splunkhecexporter/client_test.go b/exporter/splunkhecexporter/client_test.go
index 1cbb3dc8b95d..41a0b121d440 100644
--- a/exporter/splunkhecexporter/client_test.go
+++ b/exporter/splunkhecexporter/client_test.go
@@ -659,7 +659,7 @@ func Test_pushLogData_PostError(t *testing.T) {
 	assert.IsType(t, consumererror.PartialError{}, err)
 	assert.Equal(t, (err.(consumererror.PartialError)).GetLogs(), logs)
 
-	// 0 -> unlimited size batch, true -> compression enabled.
+	// 0 -> unlimited size batch, false -> compression enabled.
 	c.config.MaxContentLengthLogs, c.config.DisableCompression = 0, false
 	// 1500 < 371888 -> compression occurs.
 	c.config.MinContentLengthCompression = 1500
@@ -723,7 +723,7 @@ func TestSubLogs(t *testing.T) {
 
 	// The name of the leftmost log record should be 0_0_0.
 	assert.Equal(t, "0_0_0", got.ResourceLogs().At(0).InstrumentationLibraryLogs().At(0).Logs().At(0).Name())
-	// The name of the leftmost log record should be 1_1_2.
+	// The name of the rightmost log record should be 1_1_2.
 	assert.Equal(t, "1_1_2", got.ResourceLogs().At(1).InstrumentationLibraryLogs().At(1).Logs().At(2).Name())
 
 	// Logs subset from some mid index (resource 0, library 1, log 2).
@@ -737,10 +737,11 @@ func TestSubLogs(t *testing.T) {
 	// The name of the rightmost log record should be 1_1_2.
 	assert.Equal(t, "1_1_2", got.ResourceLogs().At(1).InstrumentationLibraryLogs().At(1).Logs().At(2).Name())
 
-	// Logs subset from some rightmost index (resource 0, library 1, log 2).
+	// Logs subset from rightmost index (resource 1, library 1, log 2).
 	_1_1_2 := &logIndex{resource: 1, library: 1, record: 2}
 	got = subLogs(&logs, _1_1_2)
 
+	// Number of logs in subset should be 1.
 	assert.Equal(t, 1, got.LogRecordCount())
 
 	// The name of the sole log record should be 1_1_2.

From a6b83f801d383f161f7f7370f721de6fc5256336 Mon Sep 17 00:00:00 2001
From: Bremer Jonathan <bjonathan@splunk.com>
Date: Fri, 26 Mar 2021 15:23:16 -0400
Subject: [PATCH 22/26] Add comment

---
 exporter/splunkhecexporter/client.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/exporter/splunkhecexporter/client.go b/exporter/splunkhecexporter/client.go
index f6572a45543d..d3449ed50eed 100644
--- a/exporter/splunkhecexporter/client.go
+++ b/exporter/splunkhecexporter/client.go
@@ -260,6 +260,7 @@ func (c *client) postEvents(ctx context.Context, events io.Reader, compressed bo
 	return nil
 }
 
+// subLogs returns a subset of `ld` starting from index `start` to the end.
 func subLogs(ld *pdata.Logs, start *logIndex) *pdata.Logs {
 	if ld == nil || start == nil || start.zero() {
 		return ld

From 36f279ae4eba5720dc70cacb2b56bfa16d95d464 Mon Sep 17 00:00:00 2001
From: Bremer Jonathan <bjonathan@splunk.com>
Date: Fri, 26 Mar 2021 16:14:11 -0400
Subject: [PATCH 23/26] Hard code minimum content length to compress

---
 exporter/splunkhecexporter/client.go      | 21 +++---
 exporter/splunkhecexporter/client_test.go | 89 ++++++++---------------
 exporter/splunkhecexporter/config.go      |  3 -
 exporter/splunkhecexporter/config_test.go | 15 ++--
 exporter/splunkhecexporter/factory.go     | 18 ++---
 5 files changed, 58 insertions(+), 88 deletions(-)

diff --git a/exporter/splunkhecexporter/client.go b/exporter/splunkhecexporter/client.go
index d3449ed50eed..f10124b58751 100644
--- a/exporter/splunkhecexporter/client.go
+++ b/exporter/splunkhecexporter/client.go
@@ -45,6 +45,9 @@ type client struct {
 	headers map[string]string
 }
 
+// Minimum number of bytes to compress. 1500 is the MTU of an ethernet frame.
+const minCompressionLen = 1500
+
 func (c *client) pushMetricsData(
 	ctx context.Context,
 	md pdata.Metrics,
@@ -57,7 +60,7 @@ func (c *client) pushMetricsData(
 		return nil
 	}
 
-	body, compressed, err := encodeBody(&c.zippers, splunkDataPoints, c.config.DisableCompression, c.config.MinContentLengthCompression)
+	body, compressed, err := encodeBody(&c.zippers, splunkDataPoints, c.config.DisableCompression)
 	if err != nil {
 		return consumererror.Permanent(err)
 	}
@@ -111,7 +114,7 @@ func (c *client) pushTraceData(
 }
 
 func (c *client) sendSplunkEvents(ctx context.Context, splunkEvents []*splunk.Event) error {
-	body, compressed, err := encodeBodyEvents(&c.zippers, splunkEvents, c.config.DisableCompression, c.config.MinContentLengthCompression)
+	body, compressed, err := encodeBodyEvents(&c.zippers, splunkEvents, c.config.DisableCompression)
 	if err != nil {
 		return consumererror.Permanent(err)
 	}
@@ -133,7 +136,7 @@ func (c *client) pushLogData(ctx context.Context, ld pdata.Logs) (err error) {
 
 	// Callback when each batch is to be sent.
 	send := func(ctx context.Context, buf *bytes.Buffer) (err error) {
-		shouldCompress := buf.Len() >= int(c.config.MinContentLengthCompression) && !c.config.DisableCompression
+		shouldCompress := buf.Len() >= minCompressionLen && !c.config.DisableCompression
 
 		if shouldCompress {
 			gzipBuffer.Reset()
@@ -302,7 +305,7 @@ func subLogs(ld *pdata.Logs, start *logIndex) *pdata.Logs {
 	return &logs
 }
 
-func encodeBodyEvents(zippers *sync.Pool, evs []*splunk.Event, disableCompression bool, minContentLengthCompression uint) (bodyReader io.Reader, compressed bool, err error) {
+func encodeBodyEvents(zippers *sync.Pool, evs []*splunk.Event, disableCompression bool) (bodyReader io.Reader, compressed bool, err error) {
 	buf := new(bytes.Buffer)
 	encoder := json.NewEncoder(buf)
 	for _, e := range evs {
@@ -312,10 +315,10 @@ func encodeBodyEvents(zippers *sync.Pool, evs []*splunk.Event, disableCompressio
 		}
 		buf.WriteString("\r\n\r\n")
 	}
-	return getReader(zippers, buf, disableCompression, minContentLengthCompression)
+	return getReader(zippers, buf, disableCompression)
 }
 
-func encodeBody(zippers *sync.Pool, dps []*splunk.Event, disableCompression bool, minContentLengthCompression uint) (bodyReader io.Reader, compressed bool, err error) {
+func encodeBody(zippers *sync.Pool, dps []*splunk.Event, disableCompression bool) (bodyReader io.Reader, compressed bool, err error) {
 	buf := new(bytes.Buffer)
 	encoder := json.NewEncoder(buf)
 	for _, e := range dps {
@@ -325,13 +328,13 @@ func encodeBody(zippers *sync.Pool, dps []*splunk.Event, disableCompression bool
 		}
 		buf.WriteString("\r\n\r\n")
 	}
-	return getReader(zippers, buf, disableCompression, minContentLengthCompression)
+	return getReader(zippers, buf, disableCompression)
 }
 
 // avoid attempting to compress things that fit into a single ethernet frame
-func getReader(zippers *sync.Pool, b *bytes.Buffer, disableCompression bool, minContentLengthCompression uint) (io.Reader, bool, error) {
+func getReader(zippers *sync.Pool, b *bytes.Buffer, disableCompression bool) (io.Reader, bool, error) {
 	var err error
-	if !disableCompression && b.Len() > int(minContentLengthCompression) {
+	if !disableCompression && b.Len() > minCompressionLen {
 		buf := new(bytes.Buffer)
 		w := zippers.Get().(*gzip.Writer)
 		defer zippers.Put(w)
diff --git a/exporter/splunkhecexporter/client_test.go b/exporter/splunkhecexporter/client_test.go
index 41a0b121d440..0227f70ab1eb 100644
--- a/exporter/splunkhecexporter/client_test.go
+++ b/exporter/splunkhecexporter/client_test.go
@@ -127,18 +127,17 @@ func createLogData(numResources int, numLibraries int, numRecords int) pdata.Log
 }
 
 type CapturingData struct {
-	testing                     *testing.T
-	receivedRequest             chan string
-	statusCode                  int
-	checkCompression            bool
-	minContentLengthCompression uint
+	testing          *testing.T
+	receivedRequest  chan string
+	statusCode       int
+	checkCompression bool
 }
 
 func (c *CapturingData) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	body, err := ioutil.ReadAll(r.Body)
 
 	if c.checkCompression {
-		if len(body) > int(c.minContentLengthCompression) && r.Header.Get("Content-Encoding") != "gzip" {
+		if len(body) > minCompressionLen && r.Header.Get("Content-Encoding") != "gzip" {
 			c.testing.Fatal("No compression")
 		}
 	}
@@ -165,7 +164,7 @@ func runMetricsExport(disableCompression bool, numberOfDataPoints int, t *testin
 	cfg.Token = "1234-1234"
 
 	receivedRequest := make(chan string)
-	capture := CapturingData{testing: t, receivedRequest: receivedRequest, statusCode: 200, checkCompression: !cfg.DisableCompression, minContentLengthCompression: cfg.MinContentLengthCompression}
+	capture := CapturingData{testing: t, receivedRequest: receivedRequest, statusCode: 200, checkCompression: !cfg.DisableCompression}
 	s := &http.Server{
 		Handler: &capture,
 	}
@@ -204,7 +203,7 @@ func runTraceExport(disableCompression bool, numberOfTraces int, t *testing.T) (
 	cfg.Token = "1234-1234"
 
 	receivedRequest := make(chan string)
-	capture := CapturingData{testing: t, receivedRequest: receivedRequest, statusCode: 200, checkCompression: !cfg.DisableCompression, minContentLengthCompression: cfg.MinContentLengthCompression}
+	capture := CapturingData{testing: t, receivedRequest: receivedRequest, statusCode: 200, checkCompression: !cfg.DisableCompression}
 	s := &http.Server{
 		Handler: &capture,
 	}
@@ -240,7 +239,7 @@ func runLogExport(cfg *Config, ld pdata.Logs, t *testing.T) ([]string, error) {
 	cfg.Token = "1234-1234"
 
 	receivedRequest := make(chan string)
-	capture := CapturingData{testing: t, receivedRequest: receivedRequest, statusCode: 200, checkCompression: !cfg.DisableCompression, minContentLengthCompression: cfg.MinContentLengthCompression}
+	capture := CapturingData{testing: t, receivedRequest: receivedRequest, statusCode: 200, checkCompression: !cfg.DisableCompression}
 	s := &http.Server{
 		Handler: &capture,
 	}
@@ -290,24 +289,19 @@ func TestReceiveLogs(t *testing.T) {
 	}
 
 	tests := []struct {
-		name func(bool) string
-		conf func(bool) *Config
+		name string
+		conf *Config
 		logs pdata.Logs
 		want wantType
 	}{
 		{
-			name: func(disableCompression bool) string {
-				return "COMPRESSION " + map[bool]string{true: "DISABLED ", false: "ENABLED "}[disableCompression] +
-					"all log events in payload when max content length unknown (configured max content length 0)"
-			},
+			name: "all log events in payload when max content length unknown (configured max content length 0)",
 			logs: createLogData(1, 1, 4),
-			conf: func(disableCompression bool) *Config {
+			conf: func() *Config {
 				cfg := NewFactory().CreateDefaultConfig().(*Config)
-				cfg.DisableCompression = disableCompression
-				cfg.MinContentLengthCompression = 4 // Given the 4 logs, 4 bytes minimum triggers compression when enable.
 				cfg.MaxContentLengthLogs = 0
 				return cfg
-			},
+			}(),
 			want: wantType{
 				batches: []string{
 					`{"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom","host.name":"myhost","service.name":"myapp"}}` + "\n\r\n\r\n" +
@@ -319,18 +313,13 @@ func TestReceiveLogs(t *testing.T) {
 			},
 		},
 		{
-			name: func(disableCompression bool) string {
-				return "COMPRESSION " + map[bool]string{true: "DISABLED ", false: "ENABLED "}[disableCompression] +
-					"1 log event per payload (configured max content length is same as event size)"
-			},
+			name: "1 log event per payload (configured max content length is same as event size)",
 			logs: createLogData(1, 1, 4),
-			conf: func(disableCompression bool) *Config {
+			conf: func() *Config {
 				cfg := NewFactory().CreateDefaultConfig().(*Config)
-				cfg.DisableCompression = disableCompression
-				cfg.MinContentLengthCompression = 4 // Given the 4 logs, 4 bytes minimum triggers compression when enable.
 				cfg.MaxContentLengthLogs = 300
 				return cfg
-			},
+			}(),
 			want: wantType{
 				batches: []string{
 					`{"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom","host.name":"myhost","service.name":"myapp"}}` + "\n\r\n\r\n",
@@ -342,18 +331,13 @@ func TestReceiveLogs(t *testing.T) {
 			},
 		},
 		{
-			name: func(disableCompression bool) string {
-				return "COMPRESSION " + map[bool]string{true: "DISABLED ", false: "ENABLED "}[disableCompression] +
-					"2 log events per payload (configured max content length is twice event size)"
-			},
+			name: "2 log events per payload (configured max content length is twice event size)",
 			logs: createLogData(1, 1, 4),
-			conf: func(disableCompression bool) *Config {
+			conf: func() *Config {
 				cfg := NewFactory().CreateDefaultConfig().(*Config)
-				cfg.DisableCompression = disableCompression
-				cfg.MinContentLengthCompression = 4 // Given the 4 logs, 4 bytes minimum triggers compression when enable.
 				cfg.MaxContentLengthLogs = 400
 				return cfg
-			},
+			}(),
 			want: wantType{
 				batches: []string{
 					`{"host":"myhost","source":"myapp","sourcetype":"myapp-type","index":"myindex","event":"mylog","fields":{"custom":"custom","host.name":"myhost","service.name":"myapp"}}` + "\n\r\n\r\n" +
@@ -367,22 +351,18 @@ func TestReceiveLogs(t *testing.T) {
 	}
 
 	for _, test := range tests {
-		for _, disabled := range []bool{true, false} {
-			t.Run(test.name(disabled), func(t *testing.T) {
-				got, err := runLogExport(test.conf(disabled), test.logs, t)
-				require.NoError(t, err)
+		t.Run(test.name, func(t *testing.T) {
+			got, err := runLogExport(test.conf, test.logs, t)
 
-				require.Len(t, got, test.want.numBatches)
+			require.NoError(t, err)
+			require.Len(t, got, test.want.numBatches)
 
-				for i := 0; i < test.want.numBatches; i++ {
-					require.NotZero(t, got[i])
+			for i := 0; i < test.want.numBatches; i++ {
+				require.NotZero(t, got[i])
+				assert.Equal(t, test.want.batches[i], got[i])
 
-					if disabled {
-						assert.Equal(t, test.want.batches[i], got[i])
-					}
-				}
-			})
-		}
+			}
+		})
 	}
 }
 
@@ -508,7 +488,7 @@ func TestInvalidJson(t *testing.T) {
 		},
 		nil,
 	}
-	reader, _, err := encodeBodyEvents(&syncPool, evs, false, defaultMinContentLengthCompression)
+	reader, _, err := encodeBodyEvents(&syncPool, evs, false)
 	assert.Error(t, err, reader)
 }
 
@@ -661,8 +641,6 @@ func Test_pushLogData_PostError(t *testing.T) {
 
 	// 0 -> unlimited size batch, false -> compression enabled.
 	c.config.MaxContentLengthLogs, c.config.DisableCompression = 0, false
-	// 1500 < 371888 -> compression occurs.
-	c.config.MinContentLengthCompression = 1500
 	err = c.pushLogData(context.Background(), logs)
 	require.Error(t, err)
 	assert.IsType(t, consumererror.PartialError{}, err)
@@ -677,8 +655,6 @@ func Test_pushLogData_PostError(t *testing.T) {
 
 	// 200000 < 371888 -> multiple batches, false -> compression enabled.
 	c.config.MaxContentLengthLogs, c.config.DisableCompression = 200000, false
-	// 1500 < 200000 -> compression occurs.
-	c.config.MinContentLengthCompression = 1500
 	err = c.pushLogData(context.Background(), logs)
 	require.Error(t, err)
 	assert.IsType(t, consumererror.PartialError{}, err)
@@ -692,12 +668,9 @@ func Test_pushLogData_Small_MaxContentLength(t *testing.T) {
 		}},
 		config: NewFactory().CreateDefaultConfig().(*Config),
 	}
-	length1byte := uint(1)
-	c.config.MinContentLengthCompression = length1byte
-	c.config.MaxContentLengthLogs = length1byte
+	c.config.MaxContentLengthLogs = 1
 
-	numLogs := 4
-	logs := createLogData(1, 1, numLogs)
+	logs := createLogData(1, 1, 2000)
 
 	for _, disable := range []bool{true, false} {
 		c.config.DisableCompression = disable
diff --git a/exporter/splunkhecexporter/config.go b/exporter/splunkhecexporter/config.go
index 108a09b1ffc5..bd0e5135e9d1 100644
--- a/exporter/splunkhecexporter/config.go
+++ b/exporter/splunkhecexporter/config.go
@@ -59,9 +59,6 @@ type Config struct {
 	// Disable GZip compression. Defaults to false.
 	DisableCompression bool `mapstructure:"disable_compression"`
 
-	// Minimum content length in bytes to compress. Defaults to 1500 bytes (Maximum Transmission Unit of an ethernet frame).
-	MinContentLengthCompression uint `mapstructure:"min_content_length_compression"`
-
 	// insecure_skip_verify skips checking the certificate of the HEC endpoint when sending data over HTTPS. Defaults to false.
 	InsecureSkipVerify bool `mapstructure:"insecure_skip_verify"`
 
diff --git a/exporter/splunkhecexporter/config_test.go b/exporter/splunkhecexporter/config_test.go
index 452c713be708..045d85ed6ceb 100644
--- a/exporter/splunkhecexporter/config_test.go
+++ b/exporter/splunkhecexporter/config_test.go
@@ -58,14 +58,13 @@ func TestLoadConfig(t *testing.T) {
 			TypeVal: configmodels.Type(typeStr),
 			NameVal: expectedName,
 		},
-		Token:                       "00000000-0000-0000-0000-0000000000000",
-		Endpoint:                    "https://splunk:8088/services/collector",
-		Source:                      "otel",
-		SourceType:                  "otel",
-		Index:                       "metrics",
-		MaxConnections:              100,
-		MinContentLengthCompression: 1500,
-		MaxContentLengthLogs:        2 * 1024 * 1024,
+		Token:                "00000000-0000-0000-0000-0000000000000",
+		Endpoint:             "https://splunk:8088/services/collector",
+		Source:               "otel",
+		SourceType:           "otel",
+		Index:                "metrics",
+		MaxConnections:       100,
+		MaxContentLengthLogs: 2 * 1024 * 1024,
 		TimeoutSettings: exporterhelper.TimeoutSettings{
 			Timeout: 10 * time.Second,
 		},
diff --git a/exporter/splunkhecexporter/factory.go b/exporter/splunkhecexporter/factory.go
index 5759556b54c8..c95ce4decac8 100644
--- a/exporter/splunkhecexporter/factory.go
+++ b/exporter/splunkhecexporter/factory.go
@@ -26,10 +26,9 @@ import (
 
 const (
 	// The value of "type" key in configuration.
-	typeStr                            = "splunk_hec"
-	defaultMaxIdleCons                 = 100
-	defaultHTTPTimeout                 = 10 * time.Second
-	defaultMinContentLengthCompression = 1500
+	typeStr            = "splunk_hec"
+	defaultMaxIdleCons = 100
+	defaultHTTPTimeout = 10 * time.Second
 )
 
 // NewFactory creates a factory for Splunk HEC exporter.
@@ -51,12 +50,11 @@ func createDefaultConfig() configmodels.Exporter {
 		TimeoutSettings: exporterhelper.TimeoutSettings{
 			Timeout: defaultHTTPTimeout,
 		},
-		RetrySettings:               exporterhelper.DefaultRetrySettings(),
-		QueueSettings:               exporterhelper.DefaultQueueSettings(),
-		DisableCompression:          false,
-		MaxConnections:              defaultMaxIdleCons,
-		MinContentLengthCompression: defaultMinContentLengthCompression,
-		MaxContentLengthLogs:        maxContentLengthLogsLimit,
+		RetrySettings:        exporterhelper.DefaultRetrySettings(),
+		QueueSettings:        exporterhelper.DefaultQueueSettings(),
+		DisableCompression:   false,
+		MaxConnections:       defaultMaxIdleCons,
+		MaxContentLengthLogs: maxContentLengthLogsLimit,
 	}
 }
 

From ad98b10beabc9122167b52580b527197721e1225 Mon Sep 17 00:00:00 2001
From: Bremer Jonathan <bjonathan@splunk.com>
Date: Fri, 26 Mar 2021 16:23:44 -0400
Subject: [PATCH 24/26] Add comment

---
 exporter/splunkhecexporter/client.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/exporter/splunkhecexporter/client.go b/exporter/splunkhecexporter/client.go
index f10124b58751..4934d381413a 100644
--- a/exporter/splunkhecexporter/client.go
+++ b/exporter/splunkhecexporter/client.go
@@ -159,7 +159,7 @@ func (c *client) pushLogData(ctx context.Context, ld pdata.Logs) (err error) {
 	return c.pushLogDataParts(ctx, ld, send)
 }
 
-// pushLogDataParts partitions log data then pushes the parts.
+// pushLogDataParts partitions log data into parts that are less than or equal to MaxContentLengthLogs then sends the parts.
 func (c *client) pushLogDataParts(ctx context.Context, ld pdata.Logs, send func(context.Context, *bytes.Buffer) error) (err error) {
 	// max is the maximum number of bytes allowed in logs buffer.
 	// pad4096 pads max by 4096 to prevent reallocation in logs buffer prior to truncation. It is assumed that a single log is likely less than 4096 bytes.

From 326b0797d08cc6bcd3c250dd9e02bf4cefa594f4 Mon Sep 17 00:00:00 2001
From: Bremer Jonathan <bjonathan@splunk.com>
Date: Tue, 30 Mar 2021 15:14:16 -0400
Subject: [PATCH 25/26] Refactor and add comments

---
 exporter/splunkhecexporter/README.md |   1 -
 exporter/splunkhecexporter/client.go | 144 +++++++++++++++------------
 2 files changed, 82 insertions(+), 63 deletions(-)

diff --git a/exporter/splunkhecexporter/README.md b/exporter/splunkhecexporter/README.md
index df3279e3f9fd..2b840034c3dc 100644
--- a/exporter/splunkhecexporter/README.md
+++ b/exporter/splunkhecexporter/README.md
@@ -23,7 +23,6 @@ The following configuration options can also be configured:
 - `timeout` (default: 10s): HTTP timeout when sending data.
 - `insecure_skip_verify` (default: false): Whether to skip checking the certificate of the HEC endpoint when sending data over HTTPS.
 - `max_content_length_logs` (default: 2097152): Maximum log data size in bytes per HTTP post limited to 2097152 bytes (2 MiB).
-- `min_content_length_compression` (default: 1500): Minimum content length in bytes to compress. 1500 is the MTU of an ethernet frame.
 
 In addition, this exporter offers queued retry which is enabled by default.
 Information about queued retry configuration parameters can be found
diff --git a/exporter/splunkhecexporter/client.go b/exporter/splunkhecexporter/client.go
index 4934d381413a..1b2ce030433b 100644
--- a/exporter/splunkhecexporter/client.go
+++ b/exporter/splunkhecexporter/client.go
@@ -156,74 +156,91 @@ func (c *client) pushLogData(ctx context.Context, ld pdata.Logs) (err error) {
 		return c.postEvents(ctx, buf, shouldCompress)
 	}
 
-	return c.pushLogDataParts(ctx, ld, send)
+	return c.pushLogDataInBatches(ctx, ld, send)
 }
 
-// pushLogDataParts partitions log data into parts that are less than or equal to MaxContentLengthLogs then sends the parts.
-func (c *client) pushLogDataParts(ctx context.Context, ld pdata.Logs, send func(context.Context, *bytes.Buffer) error) (err error) {
-	// max is the maximum number of bytes allowed in logs buffer.
-	// pad4096 pads max by 4096 to prevent reallocation in logs buffer prior to truncation. It is assumed that a single log is likely less than 4096 bytes.
-	var max, pad4096 = c.config.MaxContentLengthLogs, uint(4096)
-	// Buffer of a part of log data.
-	var logsBuf = bytes.NewBuffer(make([]byte, 0, max+pad4096))
-	var encoder = json.NewEncoder(logsBuf)
-	// Number of bytes below max of logs in logsBuf.
-	var length int
-	// Temp storage for log over max in logsBuf.
-	var overMaxBuf = bytes.NewBuffer(make([]byte, 0, pad4096))
-	// Index of the first log in a logs part.
-	var log0Index *logIndex
+// pushLogDataInBatches sends batches of Splunk events in JSON format.
+// The batch content length is restricted to MaxContentLengthLogs.
+// ld log records are parsed to Splunk events.
+func (c *client) pushLogDataInBatches(ctx context.Context, ld pdata.Logs, send func(context.Context, *bytes.Buffer) error) (err error) {
+	// Length of retained bytes in buffer after truncation.
+	var bufLen int
+	// Buffer capacity.
+	var bufCap = c.config.MaxContentLengthLogs
+	// A guesstimated value > length of bytes of a single event.
+	// Added to buffer capacity so that buffer is likely to grow by reslicing when buf.Len() > bufCap.
+	const bufCapPadding = uint(4096)
+
+	// Buffer of JSON encoded Splunk events.
+	// Expected to grow more than bufCap then truncated to bufLen.
+	var buf = bytes.NewBuffer(make([]byte, 0, bufCap+bufCapPadding))
+	var encoder = json.NewEncoder(buf)
+
+	var tmpBuf = bytes.NewBuffer(make([]byte, 0, bufCapPadding))
+
+	// Index of the log record of the first event in buffer.
+	var bufFront *logIndex
+
 	var permanentErrors []error
-	var rls = ld.ResourceLogs()
 
+	var rls = ld.ResourceLogs()
 	for i := 0; i < rls.Len(); i++ {
 		ills := rls.At(i).InstrumentationLibraryLogs()
 		for j := 0; j < ills.Len(); j++ {
 			logs := ills.At(j).Logs()
 			for k := 0; k < logs.Len(); k++ {
-				if log0Index == nil {
-					log0Index = &logIndex{resource: i, library: j, record: k}
+				if bufFront == nil {
+					bufFront = &logIndex{resource: i, library: j, record: k}
 				}
 
+				// Parsing log record to Splunk event.
 				event := mapLogRecordToSplunkEvent(logs.At(k), c.config, c.logger)
+				// JSON encoding event and writing to buffer.
 				if err = encoder.Encode(event); err != nil {
 					permanentErrors = append(permanentErrors, consumererror.Permanent(fmt.Errorf("dropped log event: %v, error: %v", event, err)))
 					continue
 				}
-				logsBuf.WriteString("\r\n\r\n")
+				buf.WriteString("\r\n\r\n")
 
-				// Consistent with ContentLength in http.Request, MaxContentLengthLogs value of 0 indicates length unknown (i.e. unbound).
-				if max == 0 || logsBuf.Len() <= int(max) {
-					length = logsBuf.Len()
+				// Continue adding events to buffer up to capacity.
+				// 0 capacity is interpreted as unknown/unbound consistent with ContentLength in http.Request.
+				if buf.Len() <= int(bufCap) || bufCap == 0 {
+					// Tracking length of event bytes below capacity in buffer.
+					bufLen = buf.Len()
 					continue
 				}
 
-				overMaxBuf.Reset()
-				if max > 0 {
-					if overLength := logsBuf.Len() - length; overLength <= int(max) {
-						overMaxBuf.Write(logsBuf.Bytes()[length:logsBuf.Len()])
+				tmpBuf.Reset()
+				// Storing event bytes over capacity in buffer before truncating.
+				if bufCap > 0 {
+					if over := buf.Len() - bufLen; over <= int(bufCap) {
+						tmpBuf.Write(buf.Bytes()[bufLen:buf.Len()])
 					} else {
-						permanentErrors = append(permanentErrors, consumererror.Permanent(fmt.Errorf("dropped log event: %s, error: event size %d bytes larger than configured max content length %d bytes", string(logsBuf.Bytes()[length:logsBuf.Len()]), overLength, max)))
+						permanentErrors = append(permanentErrors, consumererror.Permanent(
+							fmt.Errorf("dropped log event: %s, error: event size %d bytes larger than configured max content length %d bytes", string(buf.Bytes()[bufLen:buf.Len()]), over, bufCap)))
 					}
 				}
 
-				logsBuf.Truncate(length)
-				if logsBuf.Len() > 0 {
-					if err = send(ctx, logsBuf); err != nil {
-						return consumererror.PartialLogsError(err, *subLogs(&ld, log0Index))
+				// Truncating buffer at tracked length below capacity and sending.
+				buf.Truncate(bufLen)
+				if buf.Len() > 0 {
+					if err = send(ctx, buf); err != nil {
+						return consumererror.PartialLogsError(err, *subLogs(&ld, bufFront))
 					}
 				}
-				logsBuf.Reset()
-				overMaxBuf.WriteTo(logsBuf)
+				buf.Reset()
+
+				// Writing truncated bytes back to buffer.
+				tmpBuf.WriteTo(buf)
 
-				log0Index, length = nil, logsBuf.Len()
+				bufFront, bufLen = nil, buf.Len()
 			}
 		}
 	}
 
-	if logsBuf.Len() > 0 {
-		if err = send(ctx, logsBuf); err != nil {
-			return consumererror.PartialLogsError(err, *subLogs(&ld, log0Index))
+	if buf.Len() > 0 {
+		if err = send(ctx, buf); err != nil {
+			return consumererror.PartialLogsError(err, *subLogs(&ld, bufFront))
 		}
 	}
 
@@ -263,46 +280,49 @@ func (c *client) postEvents(ctx context.Context, events io.Reader, compressed bo
 	return nil
 }
 
-// subLogs returns a subset of `ld` starting from index `start` to the end.
-func subLogs(ld *pdata.Logs, start *logIndex) *pdata.Logs {
-	if ld == nil || start == nil || start.zero() {
+// subLogs returns a subset of `ld` starting from index `from` to the end.
+func subLogs(ld *pdata.Logs, from *logIndex) *pdata.Logs {
+	if ld == nil || from == nil || from.zero() {
 		return ld
 	}
 
-	logs := pdata.NewLogs()
+	subset := pdata.NewLogs()
 
-	rl, rl2 := ld.ResourceLogs(), logs.ResourceLogs()
+	resources := ld.ResourceLogs()
+	resourcesSub := subset.ResourceLogs()
 
-	for r := start.resource; r < rl.Len(); r++ {
-		rl2.Append(pdata.NewResourceLogs())
-		rl.At(r).Resource().CopyTo(rl2.At(r - start.resource).Resource())
+	for i := from.resource; i < resources.Len(); i++ {
+		resourcesSub.Append(pdata.NewResourceLogs())
+		resources.At(i).Resource().CopyTo(resourcesSub.At(i - from.resource).Resource())
 
-		il, il2 := rl.At(r).InstrumentationLibraryLogs(), rl2.At(r-start.resource).InstrumentationLibraryLogs()
+		libraries := resources.At(i).InstrumentationLibraryLogs()
+		librariesSub := resourcesSub.At(i - from.resource).InstrumentationLibraryLogs()
 
-		i := 0
-		if r == start.resource {
-			i = start.library
+		j := 0
+		if i == from.resource {
+			j = from.library
 		}
-		for i2 := 0; i < il.Len(); i++ {
-			il2.Append(pdata.NewInstrumentationLibraryLogs())
-			il.At(i).InstrumentationLibrary().CopyTo(il2.At(i2).InstrumentationLibrary())
+		for jSub := 0; j < libraries.Len(); j++ {
+			librariesSub.Append(pdata.NewInstrumentationLibraryLogs())
+			libraries.At(j).InstrumentationLibrary().CopyTo(librariesSub.At(jSub).InstrumentationLibrary())
 
-			lr, lr2 := il.At(i).Logs(), il2.At(i2).Logs()
-			i2++
+			logs := libraries.At(j).Logs()
+			logsSub := librariesSub.At(jSub).Logs()
+			jSub++
 
-			l := 0
-			if r == start.resource && i == start.library {
-				l = start.record
+			k := 0
+			if i == from.resource && j == from.library {
+				k = from.record
 			}
-			for l2 := 0; l < lr.Len(); l++ {
-				lr2.Append(pdata.NewLogRecord())
-				lr.At(l).CopyTo(lr2.At(l2))
-				l2++
+			for kSub := 0; k < logs.Len(); k++ {
+				logsSub.Append(pdata.NewLogRecord())
+				logs.At(k).CopyTo(logsSub.At(kSub))
+				kSub++
 			}
 		}
 	}
 
-	return &logs
+	return &subset
 }
 
 func encodeBodyEvents(zippers *sync.Pool, evs []*splunk.Event, disableCompression bool) (bodyReader io.Reader, compressed bool, err error) {

From 1f55e5235969abf70e172c389e4839df142b17c4 Mon Sep 17 00:00:00 2001
From: Bremer Jonathan <bjonathan@splunk.com>
Date: Tue, 30 Mar 2021 15:48:51 -0400
Subject: [PATCH 26/26] Fix broken tests

---
 exporter/splunkhecexporter/client.go      |  6 +++---
 exporter/splunkhecexporter/client_test.go | 16 ++++++++--------
 exporter/splunkhecexporter/config_test.go |  2 --
 3 files changed, 11 insertions(+), 13 deletions(-)

diff --git a/exporter/splunkhecexporter/client.go b/exporter/splunkhecexporter/client.go
index 1b2ce030433b..32b3adc018d6 100644
--- a/exporter/splunkhecexporter/client.go
+++ b/exporter/splunkhecexporter/client.go
@@ -225,7 +225,7 @@ func (c *client) pushLogDataInBatches(ctx context.Context, ld pdata.Logs, send f
 				buf.Truncate(bufLen)
 				if buf.Len() > 0 {
 					if err = send(ctx, buf); err != nil {
-						return consumererror.PartialLogsError(err, *subLogs(&ld, bufFront))
+						return consumererror.NewLogs(err, *subLogs(&ld, bufFront))
 					}
 				}
 				buf.Reset()
@@ -240,11 +240,11 @@ func (c *client) pushLogDataInBatches(ctx context.Context, ld pdata.Logs, send f
 
 	if buf.Len() > 0 {
 		if err = send(ctx, buf); err != nil {
-			return consumererror.PartialLogsError(err, *subLogs(&ld, bufFront))
+			return consumererror.NewLogs(err, *subLogs(&ld, bufFront))
 		}
 	}
 
-	return consumererror.CombineErrors(permanentErrors)
+	return consumererror.Combine(permanentErrors)
 }
 
 func (c *client) postEvents(ctx context.Context, events io.Reader, compressed bool) error {
diff --git a/exporter/splunkhecexporter/client_test.go b/exporter/splunkhecexporter/client_test.go
index 0227f70ab1eb..557d8617864d 100644
--- a/exporter/splunkhecexporter/client_test.go
+++ b/exporter/splunkhecexporter/client_test.go
@@ -636,29 +636,29 @@ func Test_pushLogData_PostError(t *testing.T) {
 	c.config.MaxContentLengthLogs, c.config.DisableCompression = 0, true
 	err := c.pushLogData(context.Background(), logs)
 	require.Error(t, err)
-	assert.IsType(t, consumererror.PartialError{}, err)
-	assert.Equal(t, (err.(consumererror.PartialError)).GetLogs(), logs)
+	assert.IsType(t, consumererror.Logs{}, err)
+	assert.Equal(t, (err.(consumererror.Logs)).GetLogs(), logs)
 
 	// 0 -> unlimited size batch, false -> compression enabled.
 	c.config.MaxContentLengthLogs, c.config.DisableCompression = 0, false
 	err = c.pushLogData(context.Background(), logs)
 	require.Error(t, err)
-	assert.IsType(t, consumererror.PartialError{}, err)
-	assert.Equal(t, (err.(consumererror.PartialError)).GetLogs(), logs)
+	assert.IsType(t, consumererror.Logs{}, err)
+	assert.Equal(t, (err.(consumererror.Logs)).GetLogs(), logs)
 
 	// 200000 < 371888 -> multiple batches, true -> compression disabled.
 	c.config.MaxContentLengthLogs, c.config.DisableCompression = 200000, true
 	err = c.pushLogData(context.Background(), logs)
 	require.Error(t, err)
-	assert.IsType(t, consumererror.PartialError{}, err)
-	assert.Equal(t, (err.(consumererror.PartialError)).GetLogs(), logs)
+	assert.IsType(t, consumererror.Logs{}, err)
+	assert.Equal(t, (err.(consumererror.Logs)).GetLogs(), logs)
 
 	// 200000 < 371888 -> multiple batches, false -> compression enabled.
 	c.config.MaxContentLengthLogs, c.config.DisableCompression = 200000, false
 	err = c.pushLogData(context.Background(), logs)
 	require.Error(t, err)
-	assert.IsType(t, consumererror.PartialError{}, err)
-	assert.Equal(t, (err.(consumererror.PartialError)).GetLogs(), logs)
+	assert.IsType(t, consumererror.Logs{}, err)
+	assert.Equal(t, (err.(consumererror.Logs)).GetLogs(), logs)
 }
 
 func Test_pushLogData_Small_MaxContentLength(t *testing.T) {
diff --git a/exporter/splunkhecexporter/config_test.go b/exporter/splunkhecexporter/config_test.go
index 5bbeeb2d72e0..04890f1b69f7 100644
--- a/exporter/splunkhecexporter/config_test.go
+++ b/exporter/splunkhecexporter/config_test.go
@@ -90,7 +90,6 @@ func TestLoadConfig(t *testing.T) {
 
 func TestConfig_getOptionsFromConfig(t *testing.T) {
 	type fields struct {
-		ExporterSettings     configmodels.ExporterSettings
 		Endpoint             string
 		Token                string
 		Source               string
@@ -155,7 +154,6 @@ func TestConfig_getOptionsFromConfig(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			cfg := &Config{
-				ExporterSettings:     tt.fields.ExporterSettings,
 				Token:                tt.fields.Token,
 				Endpoint:             tt.fields.Endpoint,
 				Source:               tt.fields.Source,