[pkg/stanza] Clean up split package (#26544)

Subset of #26241 

Follows #26540 

- Rename `MultilineConfig` to `split.Config`
- Remove `Multiline`, previously a struct representation that only
wrapped a split func
- Remove `NewMultilineConfig`, because `split.Config` is just two simple
fields with "" defaults.
- Condense references in tests
- Substantially increate test coverage in `split` package.

.chloggen/pkg-stanza-rm-tokenize.yaml

@@ -0,0 +1,30 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: breaking
+
+# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver)
+component: pkg/stanza
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Rename "tokenize" package to "split"
+
+# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists.
+issues: [26540]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext: |
+ - Remove 'Multiline' struct
+ - Remove 'NewMultilineConfig' struct
+ - Rename 'MultilineConfig' to 'split.Config'
+
+# If your change doesn't affect end users or the exported elements of any package,
+# you should instead start your pull request title with [chore] or use the "Skip Changelog" label.
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: [api]

pkg/stanza/fileconsumer/config.go

@@ -20,7 +20,7 @@ import (
 	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/fileconsumer/matcher"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/operator/helper"
-	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/tokenize"
+	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/split"
 	"github.com/open-telemetry/opentelemetry-collector-contrib/pkg/stanza/trim"
 )
 
@@ -53,7 +53,6 @@ func NewConfig() *Config {
 		IncludeFileNameResolved: false,
 		IncludeFilePathResolved: false,
 		PollInterval:            200 * time.Millisecond,
-		Multiline:               tokenize.NewMultilineConfig(),
 		Encoding:                defaultEncoding,
 		StartAt:                 "end",
 		FingerprintSize:         fingerprint.DefaultSize,
@@ -66,22 +65,22 @@ func NewConfig() *Config {
 // Config is the configuration of a file input operator
 type Config struct {
 	matcher.Criteria        `mapstructure:",squash"`
-	IncludeFileName         bool                     `mapstructure:"include_file_name,omitempty"`
-	IncludeFilePath         bool                     `mapstructure:"include_file_path,omitempty"`
-	IncludeFileNameResolved bool                     `mapstructure:"include_file_name_resolved,omitempty"`
-	IncludeFilePathResolved bool                     `mapstructure:"include_file_path_resolved,omitempty"`
-	PollInterval            time.Duration            `mapstructure:"poll_interval,omitempty"`
-	StartAt                 string                   `mapstructure:"start_at,omitempty"`
-	FingerprintSize         helper.ByteSize          `mapstructure:"fingerprint_size,omitempty"`
-	MaxLogSize              helper.ByteSize          `mapstructure:"max_log_size,omitempty"`
-	MaxConcurrentFiles      int                      `mapstructure:"max_concurrent_files,omitempty"`
-	MaxBatches              int                      `mapstructure:"max_batches,omitempty"`
-	DeleteAfterRead         bool                     `mapstructure:"delete_after_read,omitempty"`
-	Multiline               tokenize.MultilineConfig `mapstructure:"multiline,omitempty"`
-	TrimConfig              trim.Config              `mapstructure:",squash,omitempty"`
-	Encoding                string                   `mapstructure:"encoding,omitempty"`
-	FlushPeriod             time.Duration            `mapstructure:"force_flush_period,omitempty"`
-	Header                  *HeaderConfig            `mapstructure:"header,omitempty"`
+	IncludeFileName         bool            `mapstructure:"include_file_name,omitempty"`
+	IncludeFilePath         bool            `mapstructure:"include_file_path,omitempty"`
+	IncludeFileNameResolved bool            `mapstructure:"include_file_name_resolved,omitempty"`
+	IncludeFilePathResolved bool            `mapstructure:"include_file_path_resolved,omitempty"`
+	PollInterval            time.Duration   `mapstructure:"poll_interval,omitempty"`
+	StartAt                 string          `mapstructure:"start_at,omitempty"`
+	FingerprintSize         helper.ByteSize `mapstructure:"fingerprint_size,omitempty"`
+	MaxLogSize              helper.ByteSize `mapstructure:"max_log_size,omitempty"`
+	MaxConcurrentFiles      int             `mapstructure:"max_concurrent_files,omitempty"`
+	MaxBatches              int             `mapstructure:"max_batches,omitempty"`
+	DeleteAfterRead         bool            `mapstructure:"delete_after_read,omitempty"`
+	SplitConfig             split.Config    `mapstructure:"multiline,omitempty"`
+	TrimConfig              trim.Config     `mapstructure:",squash,omitempty"`
+	Encoding                string          `mapstructure:"encoding,omitempty"`
+	FlushPeriod             time.Duration   `mapstructure:"force_flush_period,omitempty"`
+	Header                  *HeaderConfig   `mapstructure:"header,omitempty"`
 }
 
 type HeaderConfig struct {
@@ -101,7 +100,7 @@ func (c Config) Build(logger *zap.SugaredLogger, emit emit.Callback) (*Manager,
 	}
 
 	// Ensure that splitter is buildable
-	factory := splitter.NewMultilineFactory(c.Multiline, enc, int(c.MaxLogSize), c.TrimConfig.Func(), c.FlushPeriod)
+	factory := splitter.NewMultilineFactory(c.SplitConfig, enc, int(c.MaxLogSize), c.TrimConfig.Func(), c.FlushPeriod)
 	if _, err := factory.Build(); err != nil {
 		return nil, err
 	}