feat: support fluentbit tail offsetKey parameters

.github/workflows/lint-test.yaml

@@ -21,7 +21,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: 3.7
+          python-version: 3.9
 
       - name: Set up chart-testing
         uses: helm/chart-testing-action@v2.6.1

apis/fluentbit/v1alpha2/plugins/input/tail_types.go

@@ -33,6 +33,9 @@ type Tail struct {
 	// Set one or multiple shell patterns separated by commas to exclude files matching a certain criteria,
 	// e.g: exclude_path=*.gz,*.zip
 	ExcludePath string `json:"excludePath,omitempty"`
+	// If enabled, Fluent Bit appends the offset of the current monitored file as part of the record.
+	// The value assigned becomes the key in the map
+	OffsetKey string `json:"offsetKey,omitempty"`
 	// For new discovered files on start (without a database offset/position),
 	// read the content from the head of the file, not tail.
 	ReadFromHead *bool `json:"readFromHead,omitempty"`
@@ -124,6 +127,9 @@ func (t *Tail) Params(_ plugins.SecretLoader) (*params.KVs, error) {
 	if t.ExcludePath != "" {
 		kvs.Insert("Exclude_Path", t.ExcludePath)
 	}
+	if t.OffsetKey != "" {
+		kvs.Insert("Offset_Key", t.OffsetKey)
+	}
 	if t.ReadFromHead != nil {
 		kvs.Insert("Read_from_Head", fmt.Sprint(*t.ReadFromHead))
 	}

charts/fluent-operator/charts/fluent-bit-crds/crds/fluentbit.fluent.io_clusterinputs.yaml

@@ -761,6 +761,11 @@ spec:
                       This will help to reassembly multiline messages originally split by Docker or CRI
                       Specify one or Multiline Parser definition to apply to the content.
                     type: string
+                  offsetKey:
+                    description: |-
+                      If enabled, Fluent Bit appends the offset of the current monitored file as part of the record.
+                      The value assigned becomes the key in the map
+                    type: string
                   parser:
                     description: Specify the name of a parser to interpret the entry
                       as a structured message.

config/crd/bases/fluentbit.fluent.io_clusterinputs.yaml

@@ -761,6 +761,11 @@ spec:
                       This will help to reassembly multiline messages originally split by Docker or CRI
                       Specify one or Multiline Parser definition to apply to the content.
                     type: string
+                  offsetKey:
+                    description: |-
+                      If enabled, Fluent Bit appends the offset of the current monitored file as part of the record.
+                      The value assigned becomes the key in the map
+                    type: string
                   parser:
                     description: Specify the name of a parser to interpret the entry
                       as a structured message.

docs/plugins/fluentbit/input/tail.md

@@ -10,6 +10,7 @@ The Tail input plugin allows to monitor one or several text files. <br /> It has
 | path | Pattern specifying a specific log files or multiple ones through the use of common wildcards. | string |
 | pathKey | If enabled, it appends the name of the monitored file as part of the record. The value assigned becomes the key in the map. | string |
 | excludePath | Set one or multiple shell patterns separated by commas to exclude files matching a certain criteria, e.g: exclude_path=*.gz,*.zip | string |
+| offsetKey | If enabled, Fluent Bit appends the offset of the current monitored file as part of the record. The value assigned becomes the key in the map | string |
 | readFromHead | For new discovered files on start (without a database offset/position), read the content from the head of the file, not tail. | *bool |
 | refreshIntervalSeconds | The interval of refreshing the list of watched files in seconds. | *int64 |
 | rotateWaitSeconds | Specify the number of extra time in seconds to monitor a file once is rotated in case some pending data is flushed. | *int64 |

manifests/setup/fluent-operator-crd.yaml

@@ -2877,6 +2877,11 @@ spec:
                       This will help to reassembly multiline messages originally split by Docker or CRI
                       Specify one or Multiline Parser definition to apply to the content.
                     type: string
+                  offsetKey:
+                    description: |-
+                      If enabled, Fluent Bit appends the offset of the current monitored file as part of the record.
+                      The value assigned becomes the key in the map
+                    type: string
                   parser:
                     description: Specify the name of a parser to interpret the entry
                       as a structured message.

manifests/setup/setup.yaml

@@ -2877,6 +2877,11 @@ spec:
                       This will help to reassembly multiline messages originally split by Docker or CRI
                       Specify one or Multiline Parser definition to apply to the content.
                     type: string
+                  offsetKey:
+                    description: |-
+                      If enabled, Fluent Bit appends the offset of the current monitored file as part of the record.
+                      The value assigned becomes the key in the map
+                    type: string
                   parser:
                     description: Specify the name of a parser to interpret the entry
                       as a structured message.